Merge pull request #484 from sparkprime/go_expansion

Go expansion
pull/543/head
Dave Cunningham 9 years ago
commit a2a5564c79

@ -0,0 +1,143 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package expander
import (
"bytes"
"fmt"
"io"
"strings"
"text/template"
"github.com/Masterminds/sprig"
"github.com/cloudfoundry-incubator/candiedyaml"
"github.com/ghodss/yaml"
"github.com/kubernetes/helm/pkg/expansion"
)
// parseYAMLStream takes an encoded YAML stream and turns it into a slice of JSON-marshalable
// objects, one for each document in the stream.
func parseYAMLStream(in io.Reader) ([]interface{}, error) {
// Use candiedyaml because it's the only one that supports streams.
decoder := candiedyaml.NewDecoder(in)
var document interface{}
stream := []interface{}{}
for {
err := decoder.Decode(&document)
if err != nil {
if strings.Contains(err.Error(), "Expected document start at line") {
return stream, nil
}
return nil, err
}
// Now it's held in document but we have to do a bit of a dance to get it in a form that can
// be marshaled as JSON for our API response. The fundamental problem is that YAML is a
// superset of JSON in that it can represent non-string keys, and full IEEE floating point
// values (NaN etc). JSON only allows string keys and its definition of a number is based
// around a sequence of digits.
// Kubernetes does not make use of these features, as it uses YAML as just "pretty JSON".
// Consequently this does not affect Helm either. However, both candiedyaml and go-yaml
// return types that are too wide for JSON marshalling (i.e. map[interface{}]interface{}
// instead of map[string]interface{}), so we have to do some explicit conversion. Luckily,
// ghodss/yaml has code to help with this, since decoding from YAML to JSON-marshalable
// values is exactly the problem that it was designed to solve.
// 1) Marshal it back to YAML string.
yamlBytes, err := candiedyaml.Marshal(document)
if err != nil {
return nil, err
}
// 2) Use ghodss/yaml to unmarshal that string into JSON-compatible data structures.
var jsonObj interface{}
if err := yaml.Unmarshal(yamlBytes, &jsonObj); err != nil {
return nil, err
}
// Now it's suitable for embedding in an API response.
stream = append(stream, jsonObj)
}
}
type expander struct {
}
// NewExpander returns an Go Templating expander.
func NewExpander() expansion.Expander {
return &expander{}
}
// ExpandChart resolves the given files to a sequence of JSON-marshalable values.
func (e *expander) ExpandChart(request *expansion.ServiceRequest) (*expansion.ServiceResponse, error) {
err := expansion.ValidateRequest(request)
if err != nil {
return nil, err
}
// TODO(dcunnin): Validate via JSONschema.
chartInv := request.ChartInvocation
chartMembers := request.Chart.Members
resources := []interface{}{}
for _, file := range chartMembers {
name := file.Path
content := file.Content
tmpl := template.New(name).Funcs(sprig.HermeticTxtFuncMap())
for _, otherFile := range chartMembers {
otherName := otherFile.Path
otherContent := otherFile.Content
if name == otherName {
continue
}
_, err := tmpl.Parse(string(otherContent))
if err != nil {
return nil, err
}
}
// Have to put something in that resolves non-empty or Go templates get confused.
_, err := tmpl.Parse("# Content begins now")
if err != nil {
return nil, err
}
tmpl, err = tmpl.Parse(string(content))
if err != nil {
return nil, err
}
generated := bytes.NewBuffer(nil)
if err := tmpl.ExecuteTemplate(generated, name, chartInv.Properties); err != nil {
return nil, err
}
stream, err := parseYAMLStream(generated)
if err != nil {
return nil, fmt.Errorf("%s\nContent:\n%s", err.Error(), generated)
}
for _, doc := range stream {
resources = append(resources, doc)
}
}
return &expansion.ServiceResponse{Resources: resources}, nil
}

@ -0,0 +1,262 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package expander
import (
"fmt"
"reflect"
"strings"
"testing"
"github.com/kubernetes/helm/pkg/chart"
"github.com/kubernetes/helm/pkg/common"
"github.com/kubernetes/helm/pkg/expansion"
)
// content provides an easy way to provide file content verbatim in tests.
func content(lines []string) []byte {
return []byte(strings.Join(lines, "\n") + "\n")
}
func testExpansion(t *testing.T, req *expansion.ServiceRequest,
expResponse *expansion.ServiceResponse, expError string) {
backend := NewExpander()
response, err := backend.ExpandChart(req)
if err != nil {
message := err.Error()
if expResponse != nil || !strings.Contains(message, expError) {
t.Fatalf("unexpected error: %s\n", message)
}
} else {
if expResponse == nil {
t.Fatalf("expected error did not occur: %s\n", expError)
}
if !reflect.DeepEqual(response, expResponse) {
message := fmt.Sprintf(
"want:\n%s\nhave:\n%s\n", expResponse, response)
t.Fatalf("output mismatch:\n%s\n", message)
}
}
}
var goExpander = &chart.Expander{
Name: "GoTemplating",
Entrypoint: "templates/main.py",
}
func TestEmpty(t *testing.T) {
testExpansion(
t,
&expansion.ServiceRequest{
ChartInvocation: &common.Resource{
Name: "test_invocation",
Type: "gs://kubernetes-charts-testing/Test-1.2.3.tgz",
},
Chart: &chart.Content{
Chartfile: &chart.Chartfile{
Name: "Test",
Expander: goExpander,
},
},
},
&expansion.ServiceResponse{
Resources: []interface{}{},
},
"", // Error
)
}
func TestSingle(t *testing.T) {
testExpansion(
t,
&expansion.ServiceRequest{
ChartInvocation: &common.Resource{
Name: "test_invocation",
Type: "gs://kubernetes-charts-testing/Test-1.2.3.tgz",
},
Chart: &chart.Content{
Chartfile: &chart.Chartfile{
Name: "Test",
Expander: goExpander,
},
Members: []*chart.Member{
{
Path: "templates/main.yaml",
Content: content([]string{
"name: foo",
"type: bar",
}),
},
},
},
},
&expansion.ServiceResponse{
Resources: []interface{}{
map[string]interface{}{
"name": "foo",
"type": "bar",
},
},
},
"", // Error
)
}
func TestProperties(t *testing.T) {
testExpansion(
t,
&expansion.ServiceRequest{
ChartInvocation: &common.Resource{
Name: "test_invocation",
Type: "gs://kubernetes-charts-testing/Test-1.2.3.tgz",
Properties: map[string]interface{}{
"prop1": 3.0,
"prop2": "foo",
},
},
Chart: &chart.Content{
Chartfile: &chart.Chartfile{
Name: "Test",
Expander: goExpander,
},
Members: []*chart.Member{
{
Path: "templates/main.yaml",
Content: content([]string{
"name: foo",
"type: {{ .prop2 }}",
"properties:",
" something: {{ .prop1 }}",
}),
},
},
},
},
&expansion.ServiceResponse{
Resources: []interface{}{
map[string]interface{}{
"name": "foo",
"properties": map[string]interface{}{
"something": 3.0,
},
"type": "foo",
},
},
},
"", // Error
)
}
func TestComplex(t *testing.T) {
testExpansion(
t,
&expansion.ServiceRequest{
ChartInvocation: &common.Resource{
Name: "test_invocation",
Type: "gs://kubernetes-charts-testing/Test-1.2.3.tgz",
Properties: map[string]interface{}{
"DatabaseName": "mydb",
"NumRepicas": 3,
},
},
Chart: &chart.Content{
Chartfile: &chart.Chartfile{
Name: "Test",
Expander: goExpander,
},
Members: []*chart.Member{
{
Path: "templates/bar.tmpl",
Content: content([]string{
`{{ template "banana" . }}`,
}),
},
{
Path: "templates/base.tmpl",
Content: content([]string{
`{{ define "apple" }}`,
`name: Abby`,
`kind: Apple`,
`dbname: {{default "whatdb" .DatabaseName}}`,
`{{ end }}`,
``,
`{{ define "banana" }}`,
`name: Bobby`,
`kind: Banana`,
`dbname: {{default "whatdb" .DatabaseName}}`,
`{{ end }}`,
}),
},
{
Path: "templates/foo.tmpl",
Content: content([]string{
`---`,
`foo:`,
` bar: baz`,
`---`,
`{{ template "apple" . }}`,
`---`,
`{{ template "apple" . }}`,
`...`,
}),
},
{
Path: "templates/docs.txt",
Content: content([]string{
`{{/*`,
`File contains only a comment.`,
`Suitable for documentation within templates/`,
`*/}}`,
}),
},
{
Path: "templates/docs2.txt",
Content: content([]string{
`# File contains only a comment.`,
`# Suitable for documentation within templates/`,
}),
},
},
},
},
&expansion.ServiceResponse{
Resources: []interface{}{
map[string]interface{}{
"name": "Bobby",
"kind": "Banana",
"dbname": "mydb",
},
map[string]interface{}{
"foo": map[string]interface{}{
"bar": "baz",
},
},
map[string]interface{}{
"name": "Abby",
"kind": "Apple",
"dbname": "mydb",
},
map[string]interface{}{
"name": "Abby",
"kind": "Apple",
"dbname": "mydb",
},
},
},
"", // Error
)
}

@ -0,0 +1,41 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"github.com/kubernetes/helm/cmd/goexpander/expander"
"github.com/kubernetes/helm/pkg/expansion"
"github.com/kubernetes/helm/pkg/version"
"flag"
"log"
)
// interface that we are going to listen on
var address = flag.String("address", "", "Interface to listen on")
// port that we are going to listen on
var port = flag.Int("port", 8080, "Port to listen on")
func main() {
flag.Parse()
backend := expander.NewExpander()
service := expansion.NewService(*address, *port, backend)
log.Printf("Version: %s", version.Version)
log.Printf("Listening on http://%s:%d/expand", *address, *port)
log.Fatal(service.ListenAndServe())
}

11
glide.lock generated

@ -1,8 +1,10 @@
hash: 5a891985ff2f22aed4eaa4f7f3819f0866750c065c7acd0118415c34788bc665
updated: 2016-03-30T15:52:37.011995847-06:00
hash: a72f13699934fc94c8df5b56c76fc66d2279b2a4ff1396c2f1bff944456a8adf
updated: 2016-03-30T23:15:39.980396345-04:00
imports:
- name: github.com/aokoli/goutils
version: 45307ec16e3cd47cd841506c081f7afd8237d210
- name: github.com/cloudfoundry-incubator/candiedyaml
version: 479485e9bfc69ee37d074b36ce36da5e4fba7941
- name: github.com/codegangsta/cli
version: a2943485b110df8842045ae0600047f88a3a56a1
- name: github.com/emicklei/go-restful
@ -46,6 +48,10 @@ imports:
- gensupport
- googleapi
- googleapi/internal/uritemplates
- name: google.golang.org/appengine
version: a503df954af258b9a70918df2a524d6a85ecefdb
subpackages:
- urlfetch
- name: google.golang.org/cloud
version: fb10e8da373d97f6ba5e648299a10b3b91f14cd5
subpackages:
@ -55,6 +61,7 @@ imports:
version: d90005c5262a3463800497ea5a89aed5fe22c886
subpackages:
- bson
- internal/sasl
- internal/scram
- name: gopkg.in/yaml.v2
version: f7716cbe52baa25d2e9b0d0da546fcf909fc16b4

@ -7,6 +7,7 @@ import:
version: ^1.1.0
- package: github.com/aokoli/goutils
version: ^1.0.0
- package: github.com/cloudfoundry-incubator/candiedyaml
- package: github.com/codegangsta/cli
- package: github.com/emicklei/go-restful
- package: github.com/ghodss/yaml

@ -18,7 +18,7 @@ set -o pipefail
readonly ROOTFS="${DIR}/rootfs"
readonly STATIC_TARGETS=(cmd/expandybird cmd/manager cmd/resourcifier)
readonly STATIC_TARGETS=(cmd/expandybird cmd/goexpander cmd/manager cmd/resourcifier)
readonly ALL_TARGETS=(${STATIC_TARGETS[@]} cmd/helm)
error_exit() {

Loading…
Cancel
Save