diff --git a/client/client.go b/client/client.go index 779cb238e..37cc8aa18 100644 --- a/client/client.go +++ b/client/client.go @@ -16,6 +16,7 @@ package main import ( "github.com/ghodss/yaml" + "github.com/kubernetes/deployment-manager/client/registry" "github.com/kubernetes/deployment-manager/expandybird/expander" "github.com/kubernetes/deployment-manager/manager/manager" @@ -35,10 +36,11 @@ import ( ) var ( - action = flag.String("action", "deploy", "expand | deploy | list | get | delete | update | listtypes | listtypeinstances") - name = flag.String("name", "", "Name of template or deployment") - service = flag.String("service", "http://localhost:8080", "URL for deployment manager") - binary = flag.String("binary", "../expandybird/expansion/expansion.py", + action = flag.String("action", "deploy", "expand | deploy | list | get | delete | update | listtypes | listtypeinstances | types") + name = flag.String("name", "", "Name of template or deployment") + service = flag.String("service", "http://localhost:8080", "URL for deployment manager") + type_registry = flag.String("type_registry", "kubernetes/deployment-manager", "Type registry [owner/repo], defaults to kubernetes/deployment-manager/") + binary = flag.String("binary", "../expandybird/expansion/expansion.py", "Path to template expansion binary") properties = flag.String("properties", "", "Properties to use when deploying a type") @@ -57,6 +59,23 @@ func main() { flag.Parse() name := getNameArgument() switch *action { + case "types": + s := strings.Split(*type_registry, "/") + git := registry.NewGithubRegistry(s[0], s[1]) + types, err := git.List() + if err != nil { + log.Fatalf("Cannot list %v err") + } + log.Printf("Types:") + for _, t := range types { + log.Printf("%s:%s", t.Name, t.Version) + downloadURL, err := git.GetURL(t) + if err != nil { + log.Printf("Failed to get download URL for %s:%s", t.Name, t.Version) + } + log.Printf("\tdownload URL: %s", downloadURL) + } + case "expand": backend := expander.NewExpander(*binary) template := loadTemplate(name) diff --git a/client/registry/github_registry.go b/client/registry/github_registry.go new file mode 100644 index 000000000..e812587df --- /dev/null +++ b/client/registry/github_registry.go @@ -0,0 +1,93 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "github.com/google/go-github/github" + + "fmt" + "log" +) + +// GithubRegistry implements the Registry interface that talks to github. +type GithubRegistry struct { + owner string + repository string + client *github.Client +} + +// NewGithubRegistry creates a Registry that can be used to talk to github. +func NewGithubRegistry(owner string, repository string) *GithubRegistry { + return &GithubRegistry{ + owner: owner, + repository: repository, + client: github.NewClient(nil), + } +} + +// List the types from the Registry. +func (g *GithubRegistry) List() ([]Type, error) { + // First list all the types at the top level. + types, err := g.getDirs(TypesDir) + if err != nil { + log.Printf("Failed to list types : %v", err) + return nil, err + } + var retTypes []Type + for _, t := range types { + // Then we need to fetch the versions (directories for this type) + versions, err := g.getDirs(TypesDir + "/" + t) + if err != nil { + log.Printf("Failed to fetch versions for type: %s", t) + return nil, err + } + for _, v := range versions { + retTypes = append(retTypes, Type{Name: t, Version: v}) + } + } + + return retTypes, nil +} + +// GetURL fetches the download URL for a given Type. +func (g *GithubRegistry) GetURL(t Type) (string, error) { + _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, TypesDir+"/"+t.Name+"/"+t.Version, nil) + if err != nil { + log.Printf("Failed to list types : %v", err) + return "", err + } + for _, f := range dc { + if *f.Type == "file" { + if *f.Name == t.Name+".jinja" || *f.Name == t.Name+".py" { + return *f.DownloadURL, nil + } + } + } + return "", fmt.Errorf("Can not find type %s:%s", t.Name, t.Version) +} + +func (g *GithubRegistry) getDirs(dir string) ([]string, error) { + _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, dir, nil) + if err != nil { + log.Printf("Failed to call ListRefs : %v", err) + return nil, err + } + var dirs []string + for _, entry := range dc { + if *entry.Type == "dir" { + dirs = append(dirs, *entry.Name) + } + } + return dirs, nil +} diff --git a/client/registry/registry.go b/client/registry/registry.go new file mode 100644 index 000000000..9c18a068e --- /dev/null +++ b/client/registry/registry.go @@ -0,0 +1,45 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +// Registry abstracts a types registry which holds types that can be +// used in a Deployment Manager configurations. A registry root must have +// a 'types' directory which contains all the available types. Each type +// then contains version directories which in turn contains all the files +// necessary for that version of the type. +// For example a type registry holding two types: +// redis v1 (implemented in jinja) +// replicatedservice v2 (implemented in python) +// would have a directory structure like so: +// /types/redis/v1 +// redis.jinja +// redis.jinja.schema +// /types/replicatedservice/v2 +// replicatedservice.python +// replicatedservice.python.schema + +const TypesDir string = "types" + +type Type struct { + Name string + Version string +} + +// Registry abstracts type interactions. +type Registry interface { + // List all the types in the given registry + List() ([]Type, error) + // Get the download URL for a given type and version + GetURL(t Type) (string, error) +} diff --git a/types/redis/v1/redis.jinja b/types/redis/v1/redis.jinja new file mode 100644 index 000000000..9f9967cf4 --- /dev/null +++ b/types/redis/v1/redis.jinja @@ -0,0 +1,32 @@ +{% set REDIS_PORT = 6379 %} +{% set WORKERS = properties['workers'] or 2 %} + +resources: +- name: redis-master + type: https://raw.githubusercontent.com/kubernetes/deployment-manager/master/examples/replicatedservice/replicatedservice.py + properties: + # This has to be overwritten since service names are hard coded in the code + service_name: redis-master + service_port: {{ REDIS_PORT }} + target_port: {{ REDIS_PORT }} + container_port: {{ REDIS_PORT }} + replicas: 1 + container_name: master + image: redis + +- name: redis-slave + type: https://raw.githubusercontent.com/kubernetes/deployment-manager/master/examples/replicatedservice/replicatedservice.py + properties: + # This has to be overwritten since service names are hard coded in the code + service_name: redis-slave + service_port: {{ REDIS_PORT }} + container_port: {{ REDIS_PORT }} + replicas: {{ WORKERS }} + container_name: worker + image: kubernetes/redis-slave:v2 + # An example of how to specify env variables. + env: + - name: GET_HOSTS_FROM + value: env + - name: REDIS_MASTER_SERVICE_HOST + value: redis-master diff --git a/types/redis/v1/redis.jinja.schema b/types/redis/v1/redis.jinja.schema new file mode 100644 index 000000000..cd550d65a --- /dev/null +++ b/types/redis/v1/redis.jinja.schema @@ -0,0 +1,10 @@ +info: + title: Redis cluster + description: Defines a redis cluster, using a single replica + replicatedservice for master and replicatedservice for workers. + +properties: + workers: + type: int + default: 2 + description: Number of worker replicas. diff --git a/types/replicatedservice/v1/replicatedservice.py b/types/replicatedservice/v1/replicatedservice.py new file mode 100644 index 000000000..72c5f20b0 --- /dev/null +++ b/types/replicatedservice/v1/replicatedservice.py @@ -0,0 +1,169 @@ +"""Defines a ReplicatedService type by creating both a Service and an RC. + +This module creates a typical abstraction for running a service in a +Kubernetes cluster, namely a replication controller and a service packaged +together into a single unit. +""" + +import yaml + +SERVICE_TYPE_COLLECTION = 'Service' +RC_TYPE_COLLECTION = 'ReplicationController' + + +def GenerateConfig(context): + """Generates a Replication Controller and a matching Service. + + Args: + context: Template context. See schema for context properties. + + Returns: + A Container Manifest as a YAML string. + """ + # YAML config that we're going to create for both RC & Service + config = {'resources': []} + + name = context.env['name'] + container_name = context.properties.get('container_name', name) + namespace = context.properties.get('namespace', 'default') + + # Define things that the Service cares about + service_name = context.properties.get('service_name', name + '-service') + service_type = SERVICE_TYPE_COLLECTION + + # Define things that the Replication Controller (rc) cares about + rc_name = name + '-rc' + rc_type = RC_TYPE_COLLECTION + + service = { + 'name': service_name, + 'type': service_type, + 'properties': { + 'apiVersion': 'v1', + 'kind': 'Service', + 'namespace': namespace, + 'metadata': { + 'name': service_name, + 'labels': GenerateLabels(context, service_name), + }, + 'spec': { + 'ports': [GenerateServicePorts(context, container_name)], + 'selector': GenerateLabels(context, name) + } + } + } + set_up_external_lb = context.properties.get('external_service', None) + if set_up_external_lb: + service['properties']['spec']['type'] = 'LoadBalancer' + config['resources'].append(service) + + rc = { + 'name': rc_name, + 'type': rc_type, + 'properties': { + 'apiVersion': 'v1', + 'kind': 'ReplicationController', + 'namespace': namespace, + 'metadata': { + 'name': rc_name, + 'labels': GenerateLabels(context, rc_name), + }, + 'spec': { + 'replicas': context.properties['replicas'], + 'selector': GenerateLabels(context, name), + 'template': { + 'metadata': { + 'labels': GenerateLabels(context, name), + }, + 'spec': { + 'containers': [ + { + 'env': GenerateEnv(context), + 'name': container_name, + 'image': context.properties['image'], + 'ports': [ + { + 'name': container_name, + 'containerPort': context.properties['container_port'], + } + ] + } + ] + } + } + } + } + } + + config['resources'].append(rc) + return yaml.dump(config) + + +# Generates labels either from the context.properties['labels'] or generates +# a default label 'name':name +def GenerateLabels(context, name): + """Generates labels from context.properties['labels'] or creates default. + + We make a deep copy of the context.properties['labels'] section to avoid + linking in the yaml document, which I believe reduces readability of the + expanded template. If no labels are given, generate a default 'name':name. + + Args: + context: Template context, which can contain the following properties: + labels - Labels to generate + + Returns: + A dict containing labels in a name:value format + """ + tmp_labels = context.properties.get('labels', None) + ret_labels = {'name': name} + if isinstance(tmp_labels, dict): + for key, value in tmp_labels.iteritems(): + ret_labels[key] = value + return ret_labels + + +def GenerateServicePorts(context, name): + """Generates a ports section for a service. + + Args: + context: Template context, which can contain the following properties: + service_port - Port to use for the service + target_port - Target port for the service + protocol - Protocol to use. + + Returns: + A dict containing a port definition + """ + service_port = context.properties.get('service_port', None) + target_port = context.properties.get('target_port', None) + protocol = context.properties.get('protocol') + + ports = {} + if name: + ports['name'] = name + if service_port: + ports['port'] = service_port + if target_port: + ports['targetPort'] = target_port + if protocol: + ports['protocol'] = protocol + + return ports + +def GenerateEnv(context): + """Generates environmental variables for a pod. + + Args: + context: Template context, which can contain the following properties: + env - Environment variables to set. + + Returns: + A list containing env variables in dict format {name: 'name', value: 'value'} + """ + env = [] + tmp_env = context.properties.get('env', []) + for entry in tmp_env: + if isinstance(entry, dict): + env.append({'name': entry.get('name'), 'value': entry.get('value')}) + return env diff --git a/types/replicatedservice/v1/replicatedservice.py.schema b/types/replicatedservice/v1/replicatedservice.py.schema new file mode 100644 index 000000000..c9ce310ab --- /dev/null +++ b/types/replicatedservice/v1/replicatedservice.py.schema @@ -0,0 +1,57 @@ +info: + title: Replicated Service + description: | + Defines a ReplicatedService type by creating both a Service and an RC. + + This module creates a typical abstraction for running a service in a + Kubernetes cluster, namely a replication controller and a service packaged + together into a single unit. + +required: +- image + +properties: + container_name: + type: string + description: Name to use for container. If omitted, name is used. + service_name: + type: string + description: Name to use for service. If omitted, name-service is used. + namespace: + type: string + description: Namespace to create resources in. If omitted, 'default' is + used. + default: default + protocol: + type: string + description: Protocol to use for the service. + service_port: + type: int + description: Port to use for the service. + target_port: + type: int + description: Target port to use for the service. + container_port: + type: int + description: Port to use for the container. + replicas: + type: int + description: Number of replicas to create in RC. + image: + type: string + description: Docker image to use for replicas. + labels: + type: object + description: Labels to apply. + env: + type: object + description: Environment variables to apply. + properties: + name: + type: string + value: + type: string + external_service: + type: boolean + description: If set to true, enable external load balancer. +