Merge pull request #573 from jackgr/pluggable-services

Pluggable services
pull/575/head
Jack Greenfield 9 years ago
commit dd4263a0bb

@ -20,9 +20,7 @@ import (
"errors"
"fmt"
"log"
"net"
"net/http"
"os"
"strings"
"github.com/kubernetes/helm/cmd/manager/manager"
@ -85,17 +83,14 @@ func setupDependencies(c *router.Context) error {
return nil
}
const expanderPort = "8080"
const deployerPort = "8080"
func newManager(c *router.Context) manager.Manager {
cfg := c.Config
service := repo.NewInmemRepoService()
cp := c.CredentialProvider
rp := repo.NewRepoProvider(service, repo.NewGCSRepoProvider(cp), cp)
expander := manager.NewExpander(getServiceURL(cfg.ExpanderURL, cfg.ExpanderName, expanderPort), rp)
deployer := manager.NewDeployer(getServiceURL(cfg.DeployerURL, cfg.DeployerName, deployerPort))
address := strings.TrimPrefix(getServiceURL(cfg.MongoAddress, cfg.MongoName, cfg.MongoPort), "http://")
expander := manager.NewExpander(cfg.ExpanderPort, cfg.ExpanderURL, rp)
deployer := manager.NewDeployer(util.GetServiceURLOrDie(cfg.DeployerName, cfg.DeployerPort, cfg.DeployerURL))
address := strings.TrimPrefix(util.GetServiceURLOrDie(cfg.MongoName, cfg.MongoPort, cfg.MongoAddress), "http://")
repository := createRepository(address)
return manager.NewManager(expander, deployer, repository, rp, service, c.CredentialProvider)
}
@ -109,41 +104,6 @@ func createRepository(address string) repository.Repository {
return r
}
func getServiceURL(serviceURL, serviceName, servicePort string) string {
if serviceURL == "" {
serviceURL = makeEnvVariableURL(serviceName)
if serviceURL == "" {
addrs, err := net.LookupHost(serviceName)
if err != nil || len(addrs) < 1 {
log.Fatalf("cannot resolve service:%v. environment:%v\n", serviceName, os.Environ())
}
serviceURL = fmt.Sprintf("http://%s:%s", addrs[0], servicePort)
}
}
return serviceURL
}
// makeEnvVariableURL takes a service name and returns the value of the
// environment variable that identifies its URL, if it exists, or the empty
// string, if it doesn't.
func makeEnvVariableURL(str string) string {
prefix := makeEnvVariableName(str)
url := os.Getenv(prefix + "_PORT")
return strings.Replace(url, "tcp", "http", 1)
}
// makeEnvVariableName is copied from the Kubernetes source,
// which is referenced by the documentation for service environment variables.
func makeEnvVariableName(str string) string {
// TODO: If we simplify to "all names are DNS1123Subdomains" this
// will need two tweaks:
// 1) Handle leading digits
// 2) Handle dots
return strings.ToUpper(strings.Replace(str, "-", "_", -1))
}
func listDeploymentsHandlerFunc(w http.ResponseWriter, r *http.Request, c *router.Context) error {
handler := "manager: list deployments"
util.LogHandlerEntry(handler, r)

@ -31,9 +31,10 @@ import (
var (
port = flag.Int("port", 8080, "The port to listen on")
maxLength = flag.Int64("maxLength", 1024, "The maximum length (KB) of a template.")
expanderName = flag.String("expander", "expandybird-service", "The DNS name of the expander service.")
expanderURL = flag.String("expanderURL", "", "The URL for the expander service.")
expanderPort = flag.String("expanderPort", "8081", "The IP port of the default expander service.")
expanderURL = flag.String("expanderURL", "", "The URL for the default expander service.")
deployerName = flag.String("deployer", "resourcifier-service", "The DNS name of the deployer service.")
deployerPort = flag.String("deployerPort", "8082", "The IP port of the deployer service.")
deployerURL = flag.String("deployerURL", "", "The URL for the deployer service.")
credentialFile = flag.String("credentialFile", "", "Local file to use for credentials.")
credentialSecrets = flag.Bool("credentialSecrets", true, "Use secrets for credentials.")
@ -73,9 +74,10 @@ func parseFlags() *router.Config {
return &router.Config{
Address: fmt.Sprintf(":%d", *port),
MaxTemplateLength: *maxLength,
ExpanderName: *expanderName,
ExpanderPort: *expanderPort,
ExpanderURL: *expanderURL,
DeployerName: *deployerName,
DeployerPort: *deployerPort,
DeployerURL: *deployerURL,
CredentialFile: *credentialFile,
CredentialSecrets: *credentialSecrets,

@ -20,21 +20,17 @@ import (
"github.com/kubernetes/helm/pkg/common"
"github.com/kubernetes/helm/pkg/expansion"
"github.com/kubernetes/helm/pkg/repo"
"github.com/kubernetes/helm/pkg/util"
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
/*
const (
// TODO (iantw): Align this with a character not allowed to show up in resource names.
layoutNodeKeySeparator = "#"
)
*/
// ExpandedConfiguration is the structure returned by the expansion service.
type ExpandedConfiguration struct {
Config *common.Configuration `json:"config"`
@ -47,23 +43,20 @@ type Expander interface {
}
// NewExpander returns a new initialized Expander.
func NewExpander(URL string, rp repo.IRepoProvider) Expander {
func NewExpander(port, URL string, rp repo.IRepoProvider) Expander {
if rp == nil {
rp = repo.NewRepoProvider(nil, nil, nil)
}
return &expander{expanderURL: URL, repoProvider: rp}
return &expander{expanderPort: port, expanderURL: URL, repoProvider: rp}
}
type expander struct {
repoProvider repo.IRepoProvider
expanderPort string
expanderURL string
}
func (e *expander) getBaseURL() string {
return fmt.Sprintf("%s/expand", e.expanderURL)
}
// ExpandConfiguration expands the supplied configuration and returns
// an expanded configuration.
func (e *expander) ExpandConfiguration(conf *common.Configuration) (*ExpandedConfiguration, error) {
@ -81,80 +74,81 @@ func (e *expander) expandConfiguration(conf *common.Configuration) (*ExpandedCon
// Iterate over all of the resources in the unexpanded configuration
for _, resource := range conf.Resources {
// A primitive layout resource captures only the name and type
additions := []*common.Resource{resource}
layout := &common.LayoutResource{
Resource: common.Resource{
Name: resource.Name, Type: resource.Type,
},
}
// If the type is not a chart reference, then it must be primitive
if !repo.IsChartReference(resource.Type) {
// Add it to the flat list of exapnded resources
resources = append(resources, resource)
// Add its layout to the list of layouts at this level
layouts = append(layouts, layout)
continue
}
// It is a chart, so go fetch it, decompress it and unpack it
cbr, _, err := e.repoProvider.GetChartByReference(resource.Type)
if err != nil {
return nil, err
}
defer cbr.Close()
// Now, load the charts contents into strings that we can pass to exapnsion
content, err := cbr.LoadContent()
if err != nil {
return nil, err
}
// Build a request to the expansion service and call it to do the expansion
svcReq := &expansion.ServiceRequest{
ChartInvocation: resource,
Chart: content,
}
svcResp, err := e.callService(svcReq)
if err != nil {
return nil, err
// If the type is a chart reference
if repo.IsChartReference(resource.Type) {
// Fetch, decompress and unpack
cbr, _, err := e.repoProvider.GetChartByReference(resource.Type)
if err != nil {
return nil, err
}
defer cbr.Close()
expander := cbr.Chartfile().Expander
if expander != nil && expander.Name != "" {
// Load the charts contents into strings that we can pass to exapnsion
content, err := cbr.LoadContent()
if err != nil {
return nil, err
}
// Build a request to the expansion service and call it to do the expansion
svcReq := &expansion.ServiceRequest{
ChartInvocation: resource,
Chart: content,
}
svcResp, err := e.callService(expander.Name, svcReq)
if err != nil {
return nil, err
}
// Call ourselves recursively with the list of resources returned by expansion
expConf, err := e.expandConfiguration(svcResp)
if err != nil {
return nil, err
}
// Append the reources returned by the recursion to the flat list of resources
additions = expConf.Config.Resources
// This was not a primitive resource, so add its properties to the layout
// Then add the all of the layout resources returned by the recursion to the layout
layout.Properties = resource.Properties
layout.Resources = expConf.Layout.Resources
}
}
// Call ourselves recursively with the list of resources returned by expansion
expConf, err := e.expandConfiguration(svcResp)
if err != nil {
return nil, err
}
// Append the reources returned by the recursion to the flat list of resources
resources = append(resources, expConf.Config.Resources...)
// This was not a primitive resource, so add its properties to the layout
layout.Properties = resource.Properties
// Now add the all of the layout resources returned by the recursion to the layout
layout.Resources = expConf.Layout.Resources
resources = append(resources, additions...)
layouts = append(layouts, layout)
}
// All done with this level, so return the espanded configuration
// All done with this level, so return the expanded configuration
return &ExpandedConfiguration{
Config: &common.Configuration{Resources: resources},
Layout: &common.Layout{Resources: layouts},
}, nil
}
func (e *expander) callService(svcReq *expansion.ServiceRequest) (*common.Configuration, error) {
func (e *expander) callService(svcName string, svcReq *expansion.ServiceRequest) (*common.Configuration, error) {
svcURL, err := e.getServiceURL(svcName)
if err != nil {
return nil, err
}
j, err := json.Marshal(svcReq)
if err != nil {
return nil, err
}
reader := ioutil.NopCloser(bytes.NewReader(j))
request, err := http.NewRequest("POST", e.getBaseURL(), reader)
request, err := http.NewRequest("POST", svcURL, reader)
if err != nil {
return nil, err
}
@ -188,3 +182,21 @@ func (e *expander) callService(svcReq *expansion.ServiceRequest) (*common.Config
return svcResp, nil
}
func (e *expander) getServiceURL(svcName string) (string, error) {
if !strings.HasPrefix(svcName, "http:") && !strings.HasPrefix(svcName, "https:") {
var err error
svcName, err = util.GetServiceURL(svcName, e.expanderPort, e.expanderURL)
if err != nil {
return "", err
}
}
u, err := url.Parse(svcName)
if err != nil {
return "", err
}
u.Path = fmt.Sprintf("%s/expand", u.Path)
return u.String(), nil
}

@ -27,7 +27,6 @@ import (
"testing"
"github.com/ghodss/yaml"
"github.com/kubernetes/helm/pkg/chart"
"github.com/kubernetes/helm/pkg/common"
"github.com/kubernetes/helm/pkg/expansion"
"github.com/kubernetes/helm/pkg/repo"
@ -219,21 +218,6 @@ var roundTripResponses = []*ExpandedConfiguration{
}
*/
type mockRepoProvider struct {
}
func (m *mockRepoProvider) GetChartByReference(reference string) (*chart.Chart, repo.IChartRepo, error) {
return &chart.Chart{}, nil, nil
}
func (m *mockRepoProvider) GetRepoByChartURL(URL string) (repo.IChartRepo, error) {
return nil, nil
}
func (m *mockRepoProvider) GetRepoByURL(URL string) (repo.IChartRepo, error) {
return nil, nil
}
type ExpanderTestCase struct {
Description string
Error string
@ -266,7 +250,7 @@ func TestExpandTemplate(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(etc.Handler))
defer ts.Close()
expander := NewExpander(ts.URL, getTestRepoProvider(t))
expander := NewExpander("8081", ts.URL, getTestRepoProvider(t))
resource := &common.Resource{
Name: "test_invocation",
Type: TestResourceType,

@ -31,12 +31,14 @@ type Config struct {
Address string
// MaxTemplateLength is the maximum length of a template.
MaxTemplateLength int64
// ExpanderName is the DNS name of the expansion service.
ExpanderName string
// ExpanderURL is the expander service's URL.
// ExpanderPort is the default expander's IP port
ExpanderPort string
// ExpanderURL is the default expander's URL
ExpanderURL string
// DeployerName is the deployer's DNS name
DeployerName string
// DeployerPort is the deployer's IP port
DeployerPort string
// DeployerURL is the deployer's URL
DeployerURL string
// CredentialFile is the file to the credentials.

@ -3,5 +3,5 @@ description: Port of the replicatedservice template from kubernetes/charts
version: "2"
home: ""
expander:
name: Expandybird
name: expandybird-service
entrypoint: templates/redis.jinja

@ -3,5 +3,5 @@ description: Port of the replicatedservice template from kubernetes/charts
version: "3"
home: ""
expander:
name: Expandybird
name: expandybird-service
entrypoint: templates/replicatedservice.py

@ -61,7 +61,7 @@ type EnvConstraint struct {
// Expander controls how template/ is evaluated.
type Expander struct {
// Currently just Expandybird or GoTemplate
// Kubernetes service name to look up in DNS.
Name string `json:"name"`
// During evaluation, which file to start from.
Entrypoint string `json:"entrypoint"`

@ -55,8 +55,8 @@ func TestLoadChartfile(t *testing.T) {
if expander == nil {
t.Errorf("No expander found in %s", testfile)
} else {
if expander.Name != "Expandybird" {
t.Errorf("Expected expander name Expandybird, got %s", expander.Name)
if expander.Name != "expandybird-service" {
t.Errorf("Expected expander name expandybird-service, got %s", expander.Name)
}
if expander.Entrypoint != "templates/wordpress.jinja" {

Binary file not shown.

@ -27,7 +27,7 @@ environment:
apiGroups:
- 3rdParty
expander:
name: Expandybird
name: expandybird-service
entrypoint: templates/wordpress.jinja
schema: wordpress.jinja.schema

@ -30,7 +30,7 @@ var (
TestChartName = "frobnitz"
TestChartVersion = "0.0.1"
TestArchiveName = TestChartName + "-" + TestChartVersion + ".tgz"
TestChartFile = "testdata/frobnitz/Chart.yaml"
TestChartFile = "../chart/testdata/frobnitz/Chart.yaml"
TestShouldFindRegex = regexp.MustCompile(TestArchiveName)
TestShouldNotFindRegex = regexp.MustCompile("foobar")
TestName = "bucket-name"

@ -1,6 +0,0 @@
The testdata directory here holds charts that match the specification.
The `fromnitz/` directory contains a chart that matches the chart
specification.
The `frobnitz-0.0.1.tgz` file is an archive of the `frobnitz` directory.

Binary file not shown.

@ -1,33 +0,0 @@
#helm:generate foo
name: frobnitz
description: This is a frobniz.
version: "1.2.3-alpha.1+12345"
keywords:
- frobnitz
- sprocket
- dodad
maintainers:
- name: The Helm Team
email: helm@example.com
- name: Someone Else
email: nobody@example.com
source:
- https://example.com/foo/bar
home: http://example.com
dependencies:
- name: thingerbob
location: https://example.com/charts/thingerbob-3.2.1.tgz
version: ^3
environment:
- name: Kubernetes
version: ~1.1
extensions:
- extensions/v1beta1
- extensions/v1beta1/daemonset
apiGroups:
- 3rdParty
expander:
name: Expandybird
entrypoint: templates/wordpress.jinja
schema: wordpress.jinja.schema

@ -1 +0,0 @@
LICENSE placeholder.

@ -1,11 +0,0 @@
# Frobnitz
This is an example chart.
## Usage
This is an example. It has no usage.
## Development
For developer info, see the top-level repository.

@ -1 +0,0 @@
This is a placeholder for documentation.

@ -1,8 +0,0 @@
<?xml version="1.0"?>
<svg xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
version="1.0" width="256" height="256" id="test">
<desc>Example icon</desc>
<rect id="first" x="2" y="2" width="40" height="60" fill="navy"/>
<rect id="second" x="15" y="4" width="40" height="60" fill="red"/>
</svg>

Before

Width:  |  Height:  |  Size: 374 B

@ -1,12 +0,0 @@
# Google Cloud Deployment Manager template
resources:
- name: nfs-disk
type: compute.v1.disk
properties:
zone: us-central1-b
sizeGb: 200
- name: mysql-disk
type: compute.v1.disk
properties:
zone: us-central1-b
sizeGb: 200

@ -1,72 +0,0 @@
#helm:generate dm_template
{% set PROPERTIES = properties or {} %}
{% set PROJECT = PROPERTIES['project'] or 'dm-k8s-testing' %}
{% set NFS_SERVER = PROPERTIES['nfs-server'] or {} %}
{% set NFS_SERVER_IP = NFS_SERVER['ip'] or '10.0.253.247' %}
{% set NFS_SERVER_PORT = NFS_SERVER['port'] or 2049 %}
{% set NFS_SERVER_DISK = NFS_SERVER['disk'] or 'nfs-disk' %}
{% set NFS_SERVER_DISK_FSTYPE = NFS_SERVER['fstype'] or 'ext4' %}
{% set NGINX = PROPERTIES['nginx'] or {} %}
{% set NGINX_PORT = 80 %}
{% set NGINX_REPLICAS = NGINX['replicas'] or 2 %}
{% set WORDPRESS_PHP = PROPERTIES['wordpress-php'] or {} %}
{% set WORDPRESS_PHP_REPLICAS = WORDPRESS_PHP['replicas'] or 2 %}
{% set WORDPRESS_PHP_PORT = WORDPRESS_PHP['port'] or 9000 %}
{% set MYSQL = PROPERTIES['mysql'] or {} %}
{% set MYSQL_PORT = MYSQL['port'] or 3306 %}
{% set MYSQL_PASSWORD = MYSQL['password'] or 'mysql-password' %}
{% set MYSQL_DISK = MYSQL['disk'] or 'mysql-disk' %}
{% set MYSQL_DISK_FSTYPE = MYSQL['fstype'] or 'ext4' %}
resources:
- name: nfs
type: github.com/kubernetes/application-dm-templates/storage/nfs:v1
properties:
ip: {{ NFS_SERVER_IP }}
port: {{ NFS_SERVER_PORT }}
disk: {{ NFS_SERVER_DISK }}
fstype: {{NFS_SERVER_DISK_FSTYPE }}
- name: nginx
type: github.com/kubernetes/application-dm-templates/common/replicatedservice:v2
properties:
service_port: {{ NGINX_PORT }}
container_port: {{ NGINX_PORT }}
replicas: {{ NGINX_REPLICAS }}
external_service: true
image: gcr.io/{{ PROJECT }}/nginx:latest
volumes:
- mount_path: /var/www/html
persistentVolumeClaim:
claimName: nfs
- name: mysql
type: github.com/kubernetes/application-dm-templates/common/replicatedservice:v2
properties:
service_port: {{ MYSQL_PORT }}
container_port: {{ MYSQL_PORT }}
replicas: 1
image: mysql:5.6
env:
- name: MYSQL_ROOT_PASSWORD
value: {{ MYSQL_PASSWORD }}
volumes:
- mount_path: /var/lib/mysql
gcePersistentDisk:
pdName: {{ MYSQL_DISK }}
fsType: {{ MYSQL_DISK_FSTYPE }}
- name: wordpress-php
type: github.com/kubernetes/application-dm-templates/common/replicatedservice:v2
properties:
service_name: wordpress-php
service_port: {{ WORDPRESS_PHP_PORT }}
container_port: {{ WORDPRESS_PHP_PORT }}
replicas: 2
image: wordpress:fpm
env:
- name: WORDPRESS_DB_PASSWORD
value: {{ MYSQL_PASSWORD }}
- name: WORDPRESS_DB_HOST
value: mysql-service
volumes:
- mount_path: /var/www/html
persistentVolumeClaim:
claimName: nfs

@ -1,69 +0,0 @@
info:
title: Wordpress
description: |
Defines a Wordpress website by defining four replicated services: an NFS service, an nginx service, a wordpress-php service, and a MySQL service.
The nginx service and the Wordpress-php service both use NFS to share files.
properties:
project:
type: string
default: dm-k8s-testing
description: Project location to load the images from.
nfs-service:
type: object
properties:
ip:
type: string
default: 10.0.253.247
description: The IP of the NFS service.
port:
type: int
default: 2049
description: The port of the NFS service.
disk:
type: string
default: nfs-disk
description: The name of the persistent disk the NFS service uses.
fstype:
type: string
default: ext4
description: The filesystem the disk of the NFS service uses.
nginx:
type: object
properties:
replicas:
type: int
default: 2
description: The number of replicas for the nginx service.
wordpress-php:
type: object
properties:
replicas:
type: int
default: 2
description: The number of replicas for the wordpress-php service.
port:
type: int
default: 9000
description: The port the wordpress-php service runs on.
mysql:
type: object
properties:
port:
type: int
default: 3306
description: The port the MySQL service runs on.
password:
type: string
default: mysql-password
description: The root password of the MySQL service.
disk:
type: string
default: mysql-disk
description: The name of the persistent disk the MySQL service uses.
fstype:
type: string
default: ext4
description: The filesystem the disk of the MySQL service uses.

@ -1,6 +0,0 @@
imports:
- path: wordpress.jinja
resources:
- name: wordpress
type: wordpress.jinja

@ -16,6 +16,14 @@ limitations under the License.
package util
import (
"fmt"
"log"
"net"
"os"
"strings"
)
// KubernetesConfig defines the configuration options for talking to Kubernetes master
type KubernetesConfig struct {
KubePath string // The path to kubectl binary
@ -56,3 +64,49 @@ type KubernetesSecret struct {
Metadata map[string]string `json:"metadata"`
Data map[string]string `json:"data,omitempty"`
}
// GetServiceURL takes a service name, a service port, and a default service URL,
// and returns a URL for accessing the service. It first looks for an environment
// variable set by Kubernetes by transposing the service name. If it can't find
// one, it looks up the service name in DNS. If that doesn't work, it returns the
// default service URL. If that's empty, it returns an HTTP localhost URL for the
// service port. If service port is empty, it panics.
func GetServiceURL(serviceName, servicePort, serviceURL string) (string, error) {
if serviceName != "" {
varBase := strings.Replace(serviceName, "-", "_", -1)
varName := strings.ToUpper(varBase) + "_PORT"
serviceURL := os.Getenv(varName)
if serviceURL != "" {
return strings.Replace(serviceURL, "tcp", "http", 1), nil
}
if servicePort != "" {
addrs, err := net.LookupHost(serviceName)
if err == nil && len(addrs) > 0 {
return fmt.Sprintf("http://%s:%s", addrs[0], servicePort), nil
}
}
}
if serviceURL != "" {
return serviceURL, nil
}
if servicePort != "" {
serviceURL = fmt.Sprintf("http://localhost:%s", servicePort)
return serviceURL, nil
}
err := fmt.Errorf("cannot resolve service:%v in environment:%v\n", serviceName, os.Environ())
return "", err
}
// GetServiceURLOrDie calls GetServiceURL and exits if it returns an error.
func GetServiceURLOrDie(serviceName, servicePort, serviceURL string) string {
URL, err := GetServiceURL(serviceName, servicePort, serviceURL)
if err != nil {
log.Fatal(err)
}
return URL
}

@ -44,7 +44,7 @@ echo "Starting expandybird..."
nohup $EXPANDYBIRD > $LOGDIR/expandybird.log 2>&1 --port=8081 --expansion_binary=expansion/expansion.py &
echo "Starting deployment manager..."
nohup $MANAGER > $LOGDIR/manager.log 2>&1 --port="${MANAGER_PORT}" --kubectl="${KUBECTL}" --expanderURL=http://localhost:8081 --deployerURL=http://localhost:8082 &
nohup $MANAGER > $LOGDIR/manager.log 2>&1 --port="${MANAGER_PORT}" --kubectl="${KUBECTL}" --expanderPort=8081 --deployerPort=8082 &
if [[ "$KUBE_PROXY" ]]; then
echo "Starting kubectl proxy..."

Loading…
Cancel
Save