From 7c73cd88793b5fed1ea5efa437c07501609197dc Mon Sep 17 00:00:00 2001 From: Dave Cunningham Date: Tue, 22 Mar 2016 15:24:09 -0400 Subject: [PATCH] chartify Expansion API & expandybird service chartify create deployment API modify CLI to match --- cmd/expandybird/expander/expander.go | 161 +++++++++++----------- cmd/expandybird/expander/expander_test.go | 2 + cmd/expandybird/service/service.go | 24 ++-- cmd/expandybird/service/service_test.go | 2 + cmd/helm/deploy.go | 50 +++---- expansion/expansion.py | 4 +- pkg/chart/chart.go | 10 +- pkg/client/deployments.go | 12 +- pkg/client/deployments_test.go | 16 +-- pkg/common/types.go | 22 +++ 10 files changed, 147 insertions(+), 156 deletions(-) diff --git a/cmd/expandybird/expander/expander.go b/cmd/expandybird/expander/expander.go index 8fda42f3e..0507d9012 100644 --- a/cmd/expandybird/expander/expander.go +++ b/cmd/expandybird/expander/expander.go @@ -18,110 +18,80 @@ package expander import ( "bytes" + "encoding/json" "fmt" + "github.com/ghodss/yaml" "log" - "os" "os/exec" - "github.com/ghodss/yaml" "github.com/kubernetes/helm/pkg/common" ) -// Expander abstracts interactions with the expander and deployer services. -type Expander interface { - ExpandTemplate(template *common.Template) (string, error) -} - type expander struct { ExpansionBinary string } -// NewExpander returns a new initialized Expander. -func NewExpander(binary string) Expander { +// NewExpander returns an ExpandyBird expander. +func NewExpander(binary string) common.Expander { return &expander{binary} } -// ExpansionResult describes the unmarshalled output of ExpandTemplate. -type ExpansionResult struct { - Config map[string]interface{} - Layout map[string]interface{} +type expandyBirdConfigOutput struct { + Resources []interface{} `yaml:"resources,omitempty"` } -// NewExpansionResult creates and returns a new expansion result from -// the raw output of ExpandTemplate. -func NewExpansionResult(output string) (*ExpansionResult, error) { - eResponse := &ExpansionResult{} - if err := yaml.Unmarshal([]byte(output), eResponse); err != nil { - return nil, fmt.Errorf("cannot unmarshal expansion result (%s):\n%s", err, output) - } - - return eResponse, nil +type expandyBirdOutput struct { + Config *expandyBirdConfigOutput `yaml:"config,omitempty"` + Layout interface{} `yaml:"layout,omitempty"` } -// Marshal creates and returns an ExpansionResponse from an ExpansionResult. -func (eResult *ExpansionResult) Marshal() (*ExpansionResponse, error) { - configYaml, err := yaml.Marshal(eResult.Config) - if err != nil { - return nil, fmt.Errorf("cannot marshal manifest template (%s):\n%s", err, eResult.Config) +// ExpandChart passes the given configuration to the expander and returns the +// expanded configuration as a string on success. +func (e *expander) ExpandChart(request *common.ExpansionRequest) (*common.ExpansionResponse, error) { + if request.ChartInvocation == nil { + return nil, fmt.Errorf("Request does not have invocation field") } - - layoutYaml, err := yaml.Marshal(eResult.Layout) - if err != nil { - return nil, fmt.Errorf("cannot marshal manifest layout (%s):\n%s", err, eResult.Layout) + if request.Chart == nil { + return nil, fmt.Errorf("Request does not have chart field") } - return &ExpansionResponse{ - Config: string(configYaml), - Layout: string(layoutYaml), - }, nil -} + chartInv := request.ChartInvocation + chartFile := request.Chart.Chartfile + chartMembers := request.Chart.Members + schemaName := chartInv.Type + ".schema" -// ExpansionResponse describes the results of marshaling an ExpansionResult. -type ExpansionResponse struct { - Config string `json:"config"` - Layout string `json:"layout"` -} - -// NewExpansionResponse creates and returns a new expansion response from -// the raw output of ExpandTemplate. -func NewExpansionResponse(output string) (*ExpansionResponse, error) { - eResult, err := NewExpansionResult(output) - if err != nil { - return nil, err + if chartFile.Expander == nil { + message := fmt.Sprintf("Chart JSON does not have expander field") + return nil, fmt.Errorf("%s: %s", chartInv.Name, message) } - eResponse, err := eResult.Marshal() - if err != nil { - return nil, err + if chartFile.Expander.Name != "ExpandyBird" { + message := fmt.Sprintf("ExpandyBird cannot do this kind of expansion: ", chartFile.Expander.Name) + return nil, fmt.Errorf("%s: %s", chartInv.Name, message) } - return eResponse, nil -} - -// Unmarshal creates and returns an ExpansionResult from an ExpansionResponse. -func (eResponse *ExpansionResponse) Unmarshal() (*ExpansionResult, error) { - var config map[string]interface{} - if err := yaml.Unmarshal([]byte(eResponse.Config), &config); err != nil { - return nil, fmt.Errorf("cannot unmarshal config (%s):\n%s", err, eResponse.Config) + if e.ExpansionBinary == "" { + message := fmt.Sprintf("expansion binary cannot be empty") + return nil, fmt.Errorf("%s: %s", chartInv.Name, message) } - var layout map[string]interface{} - if err := yaml.Unmarshal([]byte(eResponse.Layout), &layout); err != nil { - return nil, fmt.Errorf("cannot unmarshal layout (%s):\n%s", err, eResponse.Layout) + entrypointIndex := -1 + schemaIndex := -1 + for i, f := range chartMembers { + if f.Path == chartFile.Expander.Entrypoint { + entrypointIndex = i + } + if f.Path == chartFile.Schema { + schemaIndex = i + } } - - return &ExpansionResult{ - Config: config, - Layout: layout, - }, nil -} - -// ExpandTemplate passes the given configuration to the expander and returns the -// expanded configuration as a string on success. -func (e *expander) ExpandTemplate(template *common.Template) (string, error) { - if e.ExpansionBinary == "" { - message := fmt.Sprintf("expansion binary cannot be empty") - return "", fmt.Errorf("error expanding template %s: %s", template.Name, message) + if entrypointIndex == -1 { + message := fmt.Sprintf("The entrypoint in the chart.yaml cannot be found: %s", chartFile.Expander.Entrypoint) + return nil, fmt.Errorf("%s: %s", chartInv.Name, message) + } + if schemaIndex == -1 { + message := fmt.Sprintf("The schema in the chart.yaml cannot be found: %s", chartFile.Schema) + return nil, fmt.Errorf("%s: %s", chartInv.Name, message) } // Those are automatically increasing buffers, so writing arbitrary large @@ -129,24 +99,42 @@ func (e *expander) ExpandTemplate(template *common.Template) (string, error) { var stdout bytes.Buffer var stderr bytes.Buffer + // Now we convert the new chart representation into the form that classic ExpandyBird takes. + + chartInvJSON, err := json.Marshal(chartInv) + if err != nil { + return nil, fmt.Errorf("error marshalling chart invocation %s: %s", chartInv.Name, err) + } + content := "{ \"resources\": [" + string(chartInvJSON) + "] }" + cmd := &exec.Cmd{ Path: e.ExpansionBinary, // Note, that binary name still has to be passed argv[0]. - Args: []string{e.ExpansionBinary, template.Content}, - // TODO(vagababov): figure out whether do we even need "PROJECT" and - // "DEPLOYMENT_NAME" variables here. - Env: append(os.Environ(), "PROJECT="+template.Name, "DEPLOYMENT_NAME="+template.Name), + Args: []string{e.ExpansionBinary, content}, Stdout: &stdout, Stderr: &stderr, } - for _, imp := range template.Imports { - cmd.Args = append(cmd.Args, imp.Name, imp.Path, imp.Content) + if chartFile.Schema != "" { + cmd.Env = []string{"VALIDATE_SCHEMA=1"} + } + + for i, f := range chartMembers { + name := f.Path + path := f.Path + if i == entrypointIndex { + // This is how expandyBird identifies the entrypoint. + name = chartInv.Type + } else if i == schemaIndex { + // Doesn't matter what it was originally called, expandyBird expects to find it here. + name = schemaName + } + cmd.Args = append(cmd.Args, name, path, string(f.Content)) } if err := cmd.Start(); err != nil { log.Printf("error starting expansion process: %s", err) - return "", err + return nil, err } cmd.Wait() @@ -154,8 +142,13 @@ func (e *expander) ExpandTemplate(template *common.Template) (string, error) { log.Printf("Expansion process: pid: %d SysTime: %v UserTime: %v", cmd.ProcessState.Pid(), cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime()) if stderr.String() != "" { - return "", fmt.Errorf("error expanding template %s: %s", template.Name, stderr.String()) + return nil, fmt.Errorf("%s: %s", chartInv.Name, stderr.String()) + } + + output := &expandyBirdOutput{} + if err := yaml.Unmarshal(stdout.Bytes(), output); err != nil { + return nil, fmt.Errorf("cannot unmarshal expansion result (%s):\n%s", err, output) } - return stdout.String(), nil + return &common.ExpansionResponse{Resources: output.Config.Resources}, nil } diff --git a/cmd/expandybird/expander/expander_test.go b/cmd/expandybird/expander/expander_test.go index 2a86b28f6..648811e4c 100644 --- a/cmd/expandybird/expander/expander_test.go +++ b/cmd/expandybird/expander/expander_test.go @@ -16,6 +16,7 @@ limitations under the License. package expander +/* import ( "fmt" "io" @@ -179,3 +180,4 @@ func TestExpandTemplate(t *testing.T) { } } } +*/ diff --git a/cmd/expandybird/service/service.go b/cmd/expandybird/service/service.go index 09e9132c6..00458f526 100644 --- a/cmd/expandybird/service/service.go +++ b/cmd/expandybird/service/service.go @@ -17,7 +17,6 @@ limitations under the License. package service import ( - "github.com/kubernetes/helm/cmd/expandybird/expander" "github.com/kubernetes/helm/pkg/common" "github.com/kubernetes/helm/pkg/util" @@ -44,8 +43,8 @@ func NewService(handler restful.RouteFunction) *Service { webService.Produces(restful.MIME_JSON, restful.MIME_XML) webService.Route(webService.POST("/expand").To(handler). Doc("Expand a template."). - Reads(&common.Template{}). - Writes(&expander.ExpansionResponse{})) + Reads(&common.ExpansionRequest{}). + Writes(&common.ExpansionResponse{})) return &Service{webService} } @@ -62,31 +61,24 @@ func (s *Service) Register(container *restful.Container) { // NewExpansionHandler returns a route function that handles an incoming // template expansion request, bound to the supplied expander. -func NewExpansionHandler(backend expander.Expander) restful.RouteFunction { +func NewExpansionHandler(backend common.Expander) restful.RouteFunction { return func(req *restful.Request, resp *restful.Response) { util.LogHandlerEntry("expandybird: expand", req.Request) - template := &common.Template{} - if err := req.ReadEntity(&template); err != nil { + request := &common.ExpansionRequest{} + if err := req.ReadEntity(&request); err != nil { logAndReturnErrorFromHandler(http.StatusBadRequest, err.Error(), resp) return } - output, err := backend.ExpandTemplate(template) + response, err := backend.ExpandChart(request) if err != nil { - message := fmt.Sprintf("error expanding template: %s", err) - logAndReturnErrorFromHandler(http.StatusBadRequest, message, resp) - return - } - - response, err := expander.NewExpansionResponse(output) - if err != nil { - message := fmt.Sprintf("error marshaling output: %s", err) + message := fmt.Sprintf("error expanding chart: %s", err) logAndReturnErrorFromHandler(http.StatusBadRequest, message, resp) return } util.LogHandlerExit("expandybird", http.StatusOK, "OK", resp.ResponseWriter) - message := fmt.Sprintf("\nConfig:\n%s\nLayout:\n%s\n", response.Config, response.Layout) + message := fmt.Sprintf("\nResources:\n%s\n", response.Resources) util.LogHandlerText("expandybird", message) resp.WriteEntity(response) } diff --git a/cmd/expandybird/service/service_test.go b/cmd/expandybird/service/service_test.go index 4f8a1bb62..030e96457 100644 --- a/cmd/expandybird/service/service_test.go +++ b/cmd/expandybird/service/service_test.go @@ -16,6 +16,7 @@ limitations under the License. package service +/* import ( "bytes" "encoding/json" @@ -223,3 +224,4 @@ func expandOutputOrDie(t *testing.T, output, description string) *expander.Expan return result } +*/ diff --git a/cmd/helm/deploy.go b/cmd/helm/deploy.go index e1cef8747..26d76ebd7 100644 --- a/cmd/helm/deploy.go +++ b/cmd/helm/deploy.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "fmt" "io/ioutil" "os" @@ -55,40 +56,29 @@ func deployCmd() cli.Command { func deploy(c *cli.Context) error { - // If there is a configuration file, use it. - cfg := &common.Configuration{} + res := &common.Resource{ + // By default + Properties: map[string]interface{}{}, + } + if c.String("config") != "" { - if err := loadConfig(cfg, c.String("config")); err != nil { + // If there is a configuration file, use it. + err := loadConfig(c.String("config"), &res.Properties) + if err != nil { return err } - } else { - cfg.Resources = []*common.Resource{ - { - Properties: map[string]interface{}{}, - }, - } } - // If there is a chart specified on the commandline, override the config - // file with it. args := c.Args() - if len(args) > 0 { - cname := args[0] - if isLocalChart(cname) { - // If we get here, we need to first package then upload the chart. - loc, err := doUpload(cname, "", c) - if err != nil { - return err - } - cfg.Resources[0].Name = loc - } else { - cfg.Resources[0].Type = cname - } + if len(args) == 0 { + return fmt.Errorf("Need chart name on commandline") } + res.Type = args[0] - // Override the name if one is passed in. if name := c.String("name"); len(name) > 0 { - cfg.Resources[0].Name = name + res.Name = name + } else { + return fmt.Errorf("Need deployed name on commandline") } if props, err := parseProperties(c.String("properties")); err != nil { @@ -98,11 +88,11 @@ func deploy(c *cli.Context) error { // knowing which resource the properties are supposed to be part // of. for n, v := range props { - cfg.Resources[0].Properties[n] = v + res.Properties[n] = v } } - return NewClient(c).PostDeployment(cfg.Resources[0].Name, cfg) + return NewClient(c).PostDeployment(res) } // isLocalChart returns true if the given path can be statted. @@ -111,11 +101,11 @@ func isLocalChart(path string) bool { return err == nil } -// loadConfig loads a file into a common.Configuration. -func loadConfig(c *common.Configuration, filename string) error { +// loadConfig loads chart arguments into c +func loadConfig(filename string, dest *map[string]interface{}) error { data, err := ioutil.ReadFile(filename) if err != nil { return err } - return yaml.Unmarshal(data, c) + return yaml.Unmarshal(data, dest) } diff --git a/expansion/expansion.py b/expansion/expansion.py index 8ede7955c..18ea7d376 100755 --- a/expansion/expansion.py +++ b/expansion/expansion.py @@ -380,8 +380,8 @@ def main(): idx += 3 env = {} - env['deployment'] = os.environ['DEPLOYMENT_NAME'] - env['project'] = os.environ['PROJECT'] + # env['deployment'] = os.environ['DEPLOYMENT_NAME'] + # env['project'] = os.environ['PROJECT'] validate_schema = 'VALIDATE_SCHEMA' in os.environ diff --git a/pkg/chart/chart.go b/pkg/chart/chart.go index bd0d567c4..fc068285a 100644 --- a/pkg/chart/chart.go +++ b/pkg/chart/chart.go @@ -427,20 +427,20 @@ func (c *Chart) loadMember(filename string) (*Member, error) { return result, nil } -//chartContent is abstraction for the contents of a chart -type chartContent struct { +// ChartContent is abstraction for the contents of a chart. +type ChartContent struct { Chartfile *Chartfile `json:"chartfile"` Members []*Member `json:"members"` } -// loadContent loads contents of a chart directory into chartContent -func (c *Chart) loadContent() (*chartContent, error) { +// loadContent loads contents of a chart directory into ChartContent +func (c *Chart) loadContent() (*ChartContent, error) { ms, err := c.loadDirectory(c.Dir()) if err != nil { return nil, err } - cc := &chartContent{ + cc := &ChartContent{ Chartfile: c.Chartfile(), Members: ms, } diff --git a/pkg/client/deployments.go b/pkg/client/deployments.go index a0790e308..9fb1e506a 100644 --- a/pkg/client/deployments.go +++ b/pkg/client/deployments.go @@ -24,7 +24,6 @@ import ( fancypath "path" "path/filepath" - "github.com/ghodss/yaml" "github.com/kubernetes/helm/pkg/common" ) @@ -102,15 +101,10 @@ func (c *Client) DeleteDeployment(name string) (*common.Deployment, error) { } // PostDeployment posts a deployment object to the manager service. -func (c *Client) PostDeployment(name string, cfg *common.Configuration) error { - d, err := yaml.Marshal(cfg) - if err != nil { - return err - } +func (c *Client) PostDeployment(res *common.Resource) error { // This is a stop-gap until we get this API cleaned up. - t := common.Template{ - Name: name, - Content: string(d), + t := common.CreateDeploymentRequest{ + ChartInvocation: res, } data, err := json.Marshal(t) diff --git a/pkg/client/deployments_test.go b/pkg/client/deployments_test.go index 34a2e381b..6f8f0c04c 100644 --- a/pkg/client/deployments_test.go +++ b/pkg/client/deployments_test.go @@ -65,15 +65,11 @@ func TestGetDeployment(t *testing.T) { } func TestPostDeployment(t *testing.T) { - cfg := &common.Configuration{ - Resources: []*common.Resource{ - { - Name: "foo", - Type: "helm:example.com/foo/bar", - Properties: map[string]interface{}{ - "port": ":8080", - }, - }, + chartInvocation := &common.Resource{ + Name: "foo", + Type: "helm:example.com/foo/bar", + Properties: map[string]interface{}{ + "port": ":8080", }, } @@ -85,7 +81,7 @@ func TestPostDeployment(t *testing.T) { } defer fc.teardown() - if err := fc.setup().PostDeployment("foo", cfg); err != nil { + if err := fc.setup().PostDeployment(chartInvocation); err != nil { t.Fatalf("failed to post deployment: %s", err) } } diff --git a/pkg/common/types.go b/pkg/common/types.go index 5b6e207b6..5d943e3dd 100644 --- a/pkg/common/types.go +++ b/pkg/common/types.go @@ -17,6 +17,7 @@ limitations under the License. package common import ( + "github.com/kubernetes/helm/pkg/chart" "time" ) @@ -97,6 +98,27 @@ type Manifest struct { Layout *Layout `json:"layout,omitempty"` } +// CreateDeploymentRequest defines the manager API to create deployments. +type CreateDeploymentRequest struct { + ChartInvocation *Resource `json:"chart_invocation"` +} + +// ExpansionRequest defines the API to expander. +type ExpansionRequest struct { + ChartInvocation *Resource `json:"chart_invocation"` + Chart *chart.ChartContent `json:"chart"` +} + +// ExpansionResponse defines the API to expander. +type ExpansionResponse struct { + Resources []interface{} `json:"resources"` +} + +// Expander abstracts interactions with the expander and deployer services. +type Expander interface { + ExpandChart(request *ExpansionRequest) (*ExpansionResponse, error) +} + // Template describes a set of resources to be deployed. // Manager expands a Template into a Configuration, which // describes the set in a form that can be instantiated.