msb-public
/
helm
mirror of https://github.com/helm/helm
1
0
Code Issues Projects Releases Wiki Activity

Refactor expander

Browse Source
pull/462/head
jackgr 9 years ago
parent 775010f576
commit ba469fbf7f
2 changed files with 161 additions and 200 deletions
Show Stats Download Patch File Download Diff File

168
cmd/manager/manager/expander.go
Unescape View File

@ -17,65 +17,53 @@ limitations under the License.
package manager package manager
import ( import (
"github.com/kubernetes/helm/pkg/common"
"github.com/kubernetes/helm/pkg/expansion"
"github.com/kubernetes/helm/pkg/repo"
"bytes" "bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"github.com/ghodss/yaml"
"github.com/kubernetes/helm/pkg/common"
) )
/*
const ( const (
// TODO (iantw): Align this with a character not allowed to show up in resource names. // TODO (iantw): Align this with a character not allowed to show up in resource names.
layoutNodeKeySeparator = "#" layoutNodeKeySeparator = "#"
) )
*/
// ExpandedTemplate is the structure returned by the expansion service. // ExpandedConfiguration is the structure returned by the expansion service.
type ExpandedTemplate struct { type ExpandedConfiguration struct {
Config *common.Configuration `json:"config"` Config *common.Configuration `json:"config"`
Layout *common.Layout `json:"layout"` Layout *common.Layout `json:"layout"`
} }
// Expander abstracts interactions with the expander and deployer services. // Expander abstracts interactions with the expander and deployer services.
type Expander interface { type Expander interface {
ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) ExpandConfiguration(conf *common.Configuration) (*ExpandedConfiguration, error)
}
// TODO: Remove mockResolver when type resolver is completely excised
type mockResolver struct {
}
func (r *mockResolver) ResolveTypes(c *common.Configuration, i []*common.ImportFile) ([]*common.ImportFile, error) {
return nil, nil
} }
// NewExpander returns a new initialized Expander. // NewExpander returns a new initialized Expander.
func NewExpander(url string) Expander { func NewExpander(URL string, rp repo.IRepoProvider) Expander {
tr := &mockResolver{} if rp == nil {
return &expander{url, tr} rp = repo.NewRepoProvider(nil, nil, nil)
}
return &expander{expanderURL: URL, repoProvider: rp}
} }
type expander struct { type expander struct {
repoProvider repo.IRepoProvider
expanderURL string expanderURL string
typeResolver TypeResolver
}
// TypeResolver finds Types in a Configuration which aren't yet reduceable to an import file
// or primitive, and attempts to replace them with a template from a URL.
type TypeResolver interface {
ResolveTypes(config *common.Configuration, imports []*common.ImportFile) ([]*common.ImportFile, error)
} }
func (e *expander) getBaseURL() string { func (e *expander) getBaseURL() string {
return fmt.Sprintf("%s/expand", e.expanderURL) return fmt.Sprintf("%s/expand", e.expanderURL)
} }
func expanderError(t *common.Template, err error) error {
return fmt.Errorf("cannot expand template named %s (%s):\n%s", t.Name, err, t.Content)
}
// ExpanderResponse gives back a layout, which has nested structure // ExpanderResponse gives back a layout, which has nested structure
// Resource0 // Resource0
// ResourceDefinition // ResourceDefinition
@ -108,6 +96,7 @@ func expanderError(t *common.Template, err error) error {
// between the name#template key to exist in the layout given a particular choice of naming. // between the name#template key to exist in the layout given a particular choice of naming.
// In practice, it would be nearly impossible to hit, but consider including properties/name/type // In practice, it would be nearly impossible to hit, but consider including properties/name/type
// into a hash of sorts to make this robust... // into a hash of sorts to make this robust...
/*
func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[string]*common.LayoutResource) map[string]*common.LayoutResource { func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[string]*common.LayoutResource) map[string]*common.LayoutResource {
ret := map[string]*common.LayoutResource{} ret := map[string]*common.LayoutResource{}
toVisit := l.Resources toVisit := l.Resources
@ -126,81 +115,68 @@ func walkLayout(l *common.Layout, imports []*common.ImportFile, toReplace map[st
return ret return ret
} }
*/
// isTemplate returns whether a given type is a template. // ExpandConfiguration expands the supplied configuration and returns
func isTemplate(t string, imports []*common.ImportFile) bool { // an expanded configuration.
for _, imp := range imports { func (e *expander) ExpandConfiguration(conf *common.Configuration) (*ExpandedConfiguration, error) {
if imp.Name == t { expConf, err := e.expandConfiguration(conf)
return true if err != nil {
} return nil, fmt.Errorf("cannot expand configuration:%s\n%v\n", err, conf)
} }
return false return expConf, nil
} }
// ExpandTemplate expands the supplied template, and returns a configuration. func (e *expander) expandConfiguration(conf *common.Configuration) (*ExpandedConfiguration, error) {
// It will also update the imports in the provided template if any were added resources := []*common.Resource{}
// during type resolution. layout := []*common.LayoutResource{}
func (e *expander) ExpandTemplate(t *common.Template) (*ExpandedTemplate, error) {
// We have a fencepost problem here.
// 1. Start by trying to resolve any missing templates
// 2. Expand the configuration using all the of the imports available to us at this point
// 3. Expansion may yield additional templates, so we run the type resolution again
// 4. If type resolution resulted in new imports being available, return to 2.
config := &common.Configuration{}
if err := yaml.Unmarshal([]byte(t.Content), config); err != nil {
e := fmt.Errorf("Unable to unmarshal configuration (%s): %s", err, t.Content)
return nil, e
}
var finalLayout *common.Layout for _, resource := range conf.Resources {
needResolve := map[string]*common.LayoutResource{} if !repo.IsChartReference(resource.Type) {
resources = append(resources, resource)
continue
}
// Start things off by attempting to resolve the templates in a first pass. cbr, _, err := e.repoProvider.GetChartByReference(resource.Type)
newImp, err := e.typeResolver.ResolveTypes(config, t.Imports)
if err != nil { if err != nil {
e := fmt.Errorf("type resolution failed: %s", err) return nil, err
return nil, expanderError(t, e)
} }
t.Imports = append(t.Imports, newImp...) defer cbr.Close()
content, err := cbr.LoadContent()
for {
// Now expand with everything imported.
result, err := e.expandTemplate(t)
if err != nil { if err != nil {
e := fmt.Errorf("template expansion: %s", err) return nil, err
return nil, expanderError(t, e)
} }
// Once we set this layout, we're operating on the "needResolve" *LayoutResources, svcReq := &expansion.ServiceRequest{
// which are pointers into the original layout structure. After each expansion we ChartInvocation: resource,
// lose the templates in the previous expansion, so we have to keep the first one Chart: content,
// around and keep appending to the pointers in it as we get more layers of expansion.
if finalLayout == nil {
finalLayout = result.Layout
} }
needResolve = walkLayout(result.Layout, t.Imports, needResolve)
newImp, err = e.typeResolver.ResolveTypes(result.Config, t.Imports) svcResp, err := e.callService(svcReq)
if err != nil { if err != nil {
e := fmt.Errorf("type resolution failed: %s", err) return nil, err
return nil, expanderError(t, e)
} }
// If the new imports contain nothing, we are done. Everything is fully expanded. expConf, err := e.expandConfiguration(svcResp)
if len(newImp) == 0 { if err != nil {
result.Layout = finalLayout return nil, err
return result, nil
} }
// Update imports with any new imports from type resolution. // TODO: build up layout hiearchically
t.Imports = append(t.Imports, newImp...) resources = append(resources, expConf.Config.Resources...)
layout = append(layout, expConf.Layout.Resources...)
} }
return &ExpandedConfiguration{
Config: &common.Configuration{Resources: resources},
Layout: &common.Layout{Resources: layout},
}, nil
} }
func (e *expander) expandTemplate(t *common.Template) (*ExpandedTemplate, error) { func (e *expander) callService(svcReq *expansion.ServiceRequest) (*common.Configuration, error) {
j, err := json.Marshal(t) j, err := json.Marshal(svcReq)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -232,37 +208,11 @@ func (e *expander) expandTemplate(t *common.Template) (*ExpandedTemplate, error)
return nil, err return nil, err
} }
er := &ExpansionResponse{} svcResp := &common.Configuration{}
if err := json.Unmarshal(body, er); err != nil { if err := json.Unmarshal(body, svcResp); err != nil {
e := fmt.Errorf("cannot unmarshal response body (%s):%s", err, body) e := fmt.Errorf("cannot unmarshal response body (%s):%s", err, body)
return nil, e return nil, e
} }
template, err := er.Unmarshal() return svcResp, nil
if err != nil {
e := fmt.Errorf("cannot unmarshal response yaml (%s):%v", err, er)
return nil, e
}
return template, nil
}
// ExpansionResponse describes the results of marshaling an ExpandedTemplate.
type ExpansionResponse struct {
Config string `json:"config"`
Layout string `json:"layout"`
}
// Unmarshal creates and returns an ExpandedTemplate from an ExpansionResponse.
func (er *ExpansionResponse) Unmarshal() (*ExpandedTemplate, error) {
template := &ExpandedTemplate{}
if err := yaml.Unmarshal([]byte(er.Config), &template.Config); err != nil {
return nil, fmt.Errorf("cannot unmarshal config (%s):\n%s", err, er.Config)
}
if err := yaml.Unmarshal([]byte(er.Layout), &template.Layout); err != nil {
return nil, fmt.Errorf("cannot unmarshal layout (%s):\n%s", err, er.Layout)
}
return template, nil
} }

175
cmd/manager/manager/expander_test.go
Unescape View File

@ -26,44 +26,24 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/ghodss/yaml"
"github.com/kubernetes/helm/pkg/chart"
"github.com/kubernetes/helm/pkg/common" "github.com/kubernetes/helm/pkg/common"
"github.com/kubernetes/helm/pkg/expansion"
"github.com/kubernetes/helm/pkg/repo"
"github.com/kubernetes/helm/pkg/util" "github.com/kubernetes/helm/pkg/util"
"github.com/ghodss/yaml"
) )
var validTemplateTestCaseData = common.Template{ var (
Name: "TestTemplate", TestRepoBucket = "kubernetes-charts-testing"
Content: string(validContentTestCaseData), TestRepoURL = "gs://" + TestRepoBucket
Imports: validImportFilesTestCaseData, TestChartName = "frobnitz"
} TestChartVersion = "0.0.1"
TestArchiveName = TestChartName + "-" + TestChartVersion + ".tgz"
var validContentTestCaseData = []byte(` TestResourceType = TestRepoURL + "/" + TestArchiveName
imports: )
- path: test-type.py
resources:
- name: test
type: test-type.py
properties:
test-property: test-value
`)
var validImportFilesTestCaseData = []*common.ImportFile{
{
Name: "test-type.py",
Content: "test-type.py validTemplateTestCaseData content",
},
{
Name: "test.py",
Content: "test.py validTemplateTestCaseData content",
},
{
Name: "test2.py",
Content: "test2.py validTemplateTestCaseData content",
},
}
var validConfigTestCaseData = []byte(` var validResponseTestCaseData = []byte(`
resources: resources:
- name: test-service - name: test-service
properties: properties:
@ -91,6 +71,7 @@ resources:
type: ReplicationController type: ReplicationController
`) `)
/*
var validLayoutTestCaseData = []byte(` var validLayoutTestCaseData = []byte(`
resources: resources:
- name: test - name: test
@ -126,15 +107,9 @@ resources:
type: test2.jinja type: test2.jinja
`) `)
var validResponseTestCaseData = ExpansionResponse{
Config: string(validConfigTestCaseData),
Layout: string(validLayoutTestCaseData),
}
var roundTripContent = ` var roundTripContent = `
config: resources:
resources: - name: test
- name: test
type: test.py type: test.py
properties: properties:
test: test test: test
@ -207,35 +182,58 @@ layout:
test: test test: test
` `
var roundTripTemplate = common.Template{ var roundTripResponse = &ExpandedConfiguration{
Name: "TestTemplate", Config: roundTripExpanded,
Content: roundTripContent, }
Imports: nil,
var roundTripResponse2 = &ExpandedConfiguration{
Config: roundTripExpanded2,
}
var roundTripResponses = []*ExpandedConfiguration{
roundTripResponse,
roundTripResponse2,
}
*/
type mockRepoProvider struct {
}
func (m *mockRepoProvider) GetChartByReference(reference string) (*chart.Chart, repo.IChartRepo, error) {
return &chart.Chart{}, nil, nil
}
func (m *mockRepoProvider) GetRepoByChartURL(URL string) (repo.IChartRepo, error) {
return nil, nil
}
func (m *mockRepoProvider) GetRepoByURL(URL string) (repo.IChartRepo, error) {
return nil, nil
} }
type ExpanderTestCase struct { type ExpanderTestCase struct {
Description string Description string
Error string Error string
Handler func(w http.ResponseWriter, r *http.Request) Handler func(w http.ResponseWriter, r *http.Request)
ValidResponse *ExpandedTemplate ValidResponse *ExpandedConfiguration
} }
func TestExpandTemplate(t *testing.T) { func TestExpandTemplate(t *testing.T) {
roundTripResponse := &ExpandedTemplate{} // roundTripResponse := &ExpandedConfiguration{}
if err := yaml.Unmarshal([]byte(finalExpanded), roundTripResponse); err != nil { // if err := yaml.Unmarshal([]byte(finalExpanded), roundTripResponse); err != nil {
panic(err) // panic(err)
} // }
tests := []ExpanderTestCase{ tests := []ExpanderTestCase{
{ {
"expect success for ExpandTemplate", "expect success for ExpandConfiguration",
"", "",
expanderSuccessHandler, expanderSuccessHandler,
getValidResponse(t, "expect success for ExpandTemplate"), getValidExpandedConfiguration(),
}, },
{ {
"expect error for ExpandTemplate", "expect error for ExpandConfiguration",
"cannot expand template", "simulated failure",
expanderErrorHandler, expanderErrorHandler,
nil, nil,
}, },
@ -245,12 +243,23 @@ func TestExpandTemplate(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(etc.Handler)) ts := httptest.NewServer(http.HandlerFunc(etc.Handler))
defer ts.Close() defer ts.Close()
expander := NewExpander(ts.URL) expander := NewExpander(ts.URL, nil)
actualResponse, err := expander.ExpandTemplate(&validTemplateTestCaseData) resource := &common.Resource{
Name: "test_invocation",
Type: TestResourceType,
}
conf := &common.Configuration{
Resources: []*common.Resource{
resource,
},
}
actualResponse, err := expander.ExpandConfiguration(conf)
if err != nil { if err != nil {
message := err.Error() message := err.Error()
if etc.Error == "" { if etc.Error == "" {
t.Errorf("Error in test case %s when there should not be.", etc.Description) t.Errorf("unexpected error in test case %s: %s", etc.Description, err)
} }
if !strings.Contains(message, etc.Error) { if !strings.Contains(message, etc.Error) {
t.Errorf("error in test case:%s:%s\n", etc.Description, message) t.Errorf("error in test case:%s:%s\n", etc.Description, message)
@ -270,42 +279,41 @@ func TestExpandTemplate(t *testing.T) {
} }
} }
func getValidResponse(t *testing.T, description string) *ExpandedTemplate { func getValidServiceResponse() *common.Configuration {
response, err := validResponseTestCaseData.Unmarshal() conf := &common.Configuration{}
if err != nil { if err := yaml.Unmarshal(validResponseTestCaseData, conf); err != nil {
t.Errorf("cannot unmarshal valid response for test case '%s': %s\n", description, err) panic(fmt.Errorf("cannot unmarshal valid response: %s\n", err))
} }
return response return conf
} }
func expanderErrorHandler(w http.ResponseWriter, r *http.Request) { func getValidExpandedConfiguration() *ExpandedConfiguration {
defer r.Body.Close() conf := getValidServiceResponse()
http.Error(w, "something failed", http.StatusInternalServerError) layout := &common.Layout{Resources: []*common.LayoutResource{}}
} return &ExpandedConfiguration{Config: conf, Layout: layout}
var roundTripResponse = ExpansionResponse{
Config: roundTripExpanded,
Layout: roundTripLayout,
} }
var roundTripResponse2 = ExpansionResponse{ func expanderErrorHandler(w http.ResponseWriter, r *http.Request) {
Config: roundTripExpanded2, defer r.Body.Close()
Layout: roundTripLayout2, http.Error(w, "simulated failure", http.StatusInternalServerError)
}
var roundTripResponses = []ExpansionResponse{
roundTripResponse,
roundTripResponse2,
} }
/*
func roundTripHandler(w http.ResponseWriter, r *http.Request) { func roundTripHandler(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close() defer r.Body.Close()
handler := "expandybird: expand" handler := "expandybird: expand"
util.LogHandlerEntry(handler, r) util.LogHandlerEntry(handler, r)
if len(roundTripResponses) < 1 {
http.Error(w, "Too many calls to round trip handler", http.StatusInternalServerError)
return
}
util.LogHandlerExitWithJSON(handler, w, roundTripResponses[0], http.StatusOK) util.LogHandlerExitWithJSON(handler, w, roundTripResponses[0], http.StatusOK)
roundTripResponses = roundTripResponses[1:] roundTripResponses = roundTripResponses[1:]
} }
*/
func expanderSuccessHandler(w http.ResponseWriter, r *http.Request) { func expanderSuccessHandler(w http.ResponseWriter, r *http.Request) {
handler := "expandybird: expand" handler := "expandybird: expand"
@ -318,19 +326,22 @@ func expanderSuccessHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
template := &common.Template{} svcReq := &expansion.ServiceRequest{}
if err := json.Unmarshal(body, template); err != nil { if err := json.Unmarshal(body, svcReq); err != nil {
status := fmt.Sprintf("cannot unmarshal request body:%s\n%s\n", err, body) status := fmt.Sprintf("cannot unmarshal request body:%s\n%s\n", err, body)
http.Error(w, status, http.StatusInternalServerError) http.Error(w, status, http.StatusInternalServerError)
return return
} }
if !reflect.DeepEqual(validTemplateTestCaseData, *template) { /*
if !reflect.DeepEqual(validRequestTestCaseData, *svcReq) {
status := fmt.Sprintf("error in http handler:\nwant:%s\nhave:%s\n", status := fmt.Sprintf("error in http handler:\nwant:%s\nhave:%s\n",
util.ToJSONOrError(validTemplateTestCaseData), util.ToJSONOrError(template)) util.ToJSONOrError(validRequestTestCaseData), util.ToJSONOrError(template))
http.Error(w, status, http.StatusInternalServerError) http.Error(w, status, http.StatusInternalServerError)
return return
} }
*/
util.LogHandlerExitWithJSON(handler, w, validResponseTestCaseData, http.StatusOK) svcResp := getValidServiceResponse()
util.LogHandlerExitWithJSON(handler, w, svcResp, http.StatusOK)
} }

Write Preview
Loading…
Cancel
Save