From e42cafa5540f3844d7f602ba05a45f7c43b2624c Mon Sep 17 00:00:00 2001 From: vaikas-google Date: Fri, 6 Nov 2015 17:51:17 -0800 Subject: [PATCH] first cut of type registry --- client/client.go | 19 +- client/registry/github_registry.go | 96 ++++++++++ client/registry/registry.go | 46 +++++ types/redis/v1/redis.jinja | 32 ++++ .../replicatedservice/v1/replicatedservice.py | 169 ++++++++++++++++++ 5 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 client/registry/github_registry.go create mode 100644 client/registry/registry.go create mode 100644 types/redis/v1/redis.jinja create mode 100644 types/replicatedservice/v1/replicatedservice.py diff --git a/client/client.go b/client/client.go index ffbc2dd32..7adb8798b 100644 --- a/client/client.go +++ b/client/client.go @@ -15,6 +15,7 @@ package main import ( "github.com/kubernetes/deployment-manager/expandybird/expander" + "github.com/kubernetes/deployment-manager/client/registry" "bytes" "encoding/json" @@ -31,9 +32,10 @@ import ( ) var ( - action = flag.String("action", "deploy", "expand | deploy | list | get | delete | update | listtypes | listtypeinstances") + action = flag.String("action", "deploy", "expand | deploy | list | get | delete | update | listtypes | listtypeinstances | types") name = flag.String("name", "", "Name of template or deployment") service = flag.String("service", "http://localhost:8080", "URL for deployment manager") + type_registry = flag.String("type_registry", "kubernetes/deployment-manager/examples/guestbook", "Type registry [owner/repo/root], defaults to kubernetes/deployment-manager/") binary = flag.String("binary", "../expandybird/expansion/expansion.py", "Path to template expansion binary") ) @@ -48,6 +50,21 @@ func main() { flag.Parse() name := getNameArgument() switch *action { + case "repolist": + git := registry.NewGithubRegistry("kubernetes", "deployment-manager", "examples/guestbook") + types, err := git.List() + if err != nil { + log.Fatalf("Cannot list %v err") + } + for _, t := range types { + log.Printf("Got type: %s:%s", t.Name, t.Version) + downloadURL, err := git.GetURL(t) + if err != nil { + log.Printf("Failed to get download URL for %s:%s", t.Name, t.Version) + } + log.Printf("DOWNLOAD URL: %s", downloadURL) + } + case "expand": backend := expander.NewExpander(*binary) template := loadTemplate(name) diff --git a/client/registry/github_registry.go b/client/registry/github_registry.go new file mode 100644 index 000000000..efc1d8079 --- /dev/null +++ b/client/registry/github_registry.go @@ -0,0 +1,96 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "github.com/google/go-github/github" + + "fmt" + "log" +) + +type GithubRegistry struct { + owner string + repository string + root string + client *github.Client +} + +func NewGithubRegistry(owner string, repository string, root string) *GithubRegistry { + return &GithubRegistry{ + owner: owner, + repository: repository, + root: root, + client: github.NewClient(nil), + } +} + +func (g *GithubRegistry) List() ([]Type, error) { + log.Printf("Calling ListRefs") + // First list all the types at the top level. + types, err := g.getDirs(TypesDir) + if err != nil { + log.Printf("Failed to list types : %v", err) + return nil, err + } + var retTypes []Type + for _, t := range types { + log.Printf("Got TYPE: %s, fetching : %s", t, TypesDir+"/"+t) + // Then we need to fetch the versions (directories for this type) + versions, err := g.getDirs(TypesDir + "/" + t) + if err != nil { + log.Printf("Failed to fetch versions for type: %s", t) + return nil, err + } + for _, v := range versions { + log.Printf("Got VERSION: %s", v) + retTypes = append(retTypes, Type{Name: t, Version: v}) + } + } + + return retTypes, nil +} + +// Get the URL for a given type +func (g *GithubRegistry) GetURL(t Type) (string, error) { + _, dc, _, err := g.client.Repositories.GetContents(g.owner, g.repository, TypesDir+"/"+t.Name+"/"+t.Version, nil) + if err != nil { + log.Printf("Failed to list types : %v", err) + return "", err + } + for _, f := range dc { + if *f.Type == "file" { + if *f.Name == t.Version+".jinja" || *f.Name == t.Version+".py" { + return *f.DownloadURL, nil + } + } + } + return "", fmt.Errorf("Can not find type %s:%s", t.Name, t.Version) +} + +func (g *GithubRegistry) getDirs(dir string) ([]string, error) { + _, dc, resp, err := g.client.Repositories.GetContents(g.owner, g.repository, dir, nil) + if err != nil { + log.Printf("Failed to call ListRefs : %v", err) + return nil, err + } + log.Printf("Got: %v %v", dc, resp, err) + var dirs []string + for _, entry := range dc { + if *entry.Type == "dir" { + dirs = append(dirs, *entry.Name) + } + } + return dirs, nil +} diff --git a/client/registry/registry.go b/client/registry/registry.go new file mode 100644 index 000000000..dba7d6d16 --- /dev/null +++ b/client/registry/registry.go @@ -0,0 +1,46 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +// Registry abstracts a types registry which holds types that can be +// used in a Deployment Manager configurations. A registry root must have +// a 'types' directory which contains all the available types. Each type +// then contains version directory which in turn contains all the files +// necessary for that type. +// For example a type registry holding two types: +// redis v1 (implemented in jinja) +// replicatedservice v2 (implemented in python) +// would have a directory structure like so: +// /types/redis/v1 +// redis.jinja +// redis.jinja.schema +// /types/replicatedservice/v2 +// replicatedservice.python +// replicatedservice.python.schema + +//const TypesDir string = "types" +const TypesDir string = "examples" + +type Type struct { + Name string + Version string +} + +// Registry abstracts type interactions. +type Registry interface { + // List all the types in the given repository + List() ([]Type, error) + // Get the download URL for a given type and version + GetURL(t Type) (string, error) +} diff --git a/types/redis/v1/redis.jinja b/types/redis/v1/redis.jinja new file mode 100644 index 000000000..9f9967cf4 --- /dev/null +++ b/types/redis/v1/redis.jinja @@ -0,0 +1,32 @@ +{% set REDIS_PORT = 6379 %} +{% set WORKERS = properties['workers'] or 2 %} + +resources: +- name: redis-master + type: https://raw.githubusercontent.com/kubernetes/deployment-manager/master/examples/replicatedservice/replicatedservice.py + properties: + # This has to be overwritten since service names are hard coded in the code + service_name: redis-master + service_port: {{ REDIS_PORT }} + target_port: {{ REDIS_PORT }} + container_port: {{ REDIS_PORT }} + replicas: 1 + container_name: master + image: redis + +- name: redis-slave + type: https://raw.githubusercontent.com/kubernetes/deployment-manager/master/examples/replicatedservice/replicatedservice.py + properties: + # This has to be overwritten since service names are hard coded in the code + service_name: redis-slave + service_port: {{ REDIS_PORT }} + container_port: {{ REDIS_PORT }} + replicas: {{ WORKERS }} + container_name: worker + image: kubernetes/redis-slave:v2 + # An example of how to specify env variables. + env: + - name: GET_HOSTS_FROM + value: env + - name: REDIS_MASTER_SERVICE_HOST + value: redis-master diff --git a/types/replicatedservice/v1/replicatedservice.py b/types/replicatedservice/v1/replicatedservice.py new file mode 100644 index 000000000..72c5f20b0 --- /dev/null +++ b/types/replicatedservice/v1/replicatedservice.py @@ -0,0 +1,169 @@ +"""Defines a ReplicatedService type by creating both a Service and an RC. + +This module creates a typical abstraction for running a service in a +Kubernetes cluster, namely a replication controller and a service packaged +together into a single unit. +""" + +import yaml + +SERVICE_TYPE_COLLECTION = 'Service' +RC_TYPE_COLLECTION = 'ReplicationController' + + +def GenerateConfig(context): + """Generates a Replication Controller and a matching Service. + + Args: + context: Template context. See schema for context properties. + + Returns: + A Container Manifest as a YAML string. + """ + # YAML config that we're going to create for both RC & Service + config = {'resources': []} + + name = context.env['name'] + container_name = context.properties.get('container_name', name) + namespace = context.properties.get('namespace', 'default') + + # Define things that the Service cares about + service_name = context.properties.get('service_name', name + '-service') + service_type = SERVICE_TYPE_COLLECTION + + # Define things that the Replication Controller (rc) cares about + rc_name = name + '-rc' + rc_type = RC_TYPE_COLLECTION + + service = { + 'name': service_name, + 'type': service_type, + 'properties': { + 'apiVersion': 'v1', + 'kind': 'Service', + 'namespace': namespace, + 'metadata': { + 'name': service_name, + 'labels': GenerateLabels(context, service_name), + }, + 'spec': { + 'ports': [GenerateServicePorts(context, container_name)], + 'selector': GenerateLabels(context, name) + } + } + } + set_up_external_lb = context.properties.get('external_service', None) + if set_up_external_lb: + service['properties']['spec']['type'] = 'LoadBalancer' + config['resources'].append(service) + + rc = { + 'name': rc_name, + 'type': rc_type, + 'properties': { + 'apiVersion': 'v1', + 'kind': 'ReplicationController', + 'namespace': namespace, + 'metadata': { + 'name': rc_name, + 'labels': GenerateLabels(context, rc_name), + }, + 'spec': { + 'replicas': context.properties['replicas'], + 'selector': GenerateLabels(context, name), + 'template': { + 'metadata': { + 'labels': GenerateLabels(context, name), + }, + 'spec': { + 'containers': [ + { + 'env': GenerateEnv(context), + 'name': container_name, + 'image': context.properties['image'], + 'ports': [ + { + 'name': container_name, + 'containerPort': context.properties['container_port'], + } + ] + } + ] + } + } + } + } + } + + config['resources'].append(rc) + return yaml.dump(config) + + +# Generates labels either from the context.properties['labels'] or generates +# a default label 'name':name +def GenerateLabels(context, name): + """Generates labels from context.properties['labels'] or creates default. + + We make a deep copy of the context.properties['labels'] section to avoid + linking in the yaml document, which I believe reduces readability of the + expanded template. If no labels are given, generate a default 'name':name. + + Args: + context: Template context, which can contain the following properties: + labels - Labels to generate + + Returns: + A dict containing labels in a name:value format + """ + tmp_labels = context.properties.get('labels', None) + ret_labels = {'name': name} + if isinstance(tmp_labels, dict): + for key, value in tmp_labels.iteritems(): + ret_labels[key] = value + return ret_labels + + +def GenerateServicePorts(context, name): + """Generates a ports section for a service. + + Args: + context: Template context, which can contain the following properties: + service_port - Port to use for the service + target_port - Target port for the service + protocol - Protocol to use. + + Returns: + A dict containing a port definition + """ + service_port = context.properties.get('service_port', None) + target_port = context.properties.get('target_port', None) + protocol = context.properties.get('protocol') + + ports = {} + if name: + ports['name'] = name + if service_port: + ports['port'] = service_port + if target_port: + ports['targetPort'] = target_port + if protocol: + ports['protocol'] = protocol + + return ports + +def GenerateEnv(context): + """Generates environmental variables for a pod. + + Args: + context: Template context, which can contain the following properties: + env - Environment variables to set. + + Returns: + A list containing env variables in dict format {name: 'name', value: 'value'} + """ + env = [] + tmp_env = context.properties.get('env', []) + for entry in tmp_env: + if isinstance(entry, dict): + env.append({'name': entry.get('name'), 'value': entry.get('value')}) + return env