Add new repository provider

pkg/repo/inmem_credential_provider.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package repo
+
+import (
+	"fmt"
+)
+
+// InmemCredentialProvider is a memory based credential provider.
+type InmemCredentialProvider struct {
+	credentials map[string]*RepoCredential
+}
+
+// NewInmemCredentialProvider creates a new memory based credential provider.
+func NewInmemCredentialProvider() CredentialProvider {
+	return &InmemCredentialProvider{credentials: make(map[string]*RepoCredential)}
+}
+
+// GetCredential returns a credential by name.
+func (fcp *InmemCredentialProvider) GetCredential(name string) (*RepoCredential, error) {
+	if val, ok := fcp.credentials[name]; ok {
+		return val, nil
+	}
+	return nil, fmt.Errorf("no such credential: %s", name)
+}
+
+// SetCredential sets a credential by name.
+func (fcp *InmemCredentialProvider) SetCredential(name string, credential *RepoCredential) error {
+	fcp.credentials[name] = &RepoCredential{APIToken: credential.APIToken, BasicAuth: credential.BasicAuth, ServiceAccount: credential.ServiceAccount}
+	return nil
+}

pkg/repo/inmem_credential_provider_test.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package repo
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+type testCase struct {
+	name   string
+	exp    *RepoCredential
+	expErr error
+}
+
+func createMissingError(name string) error {
+	return fmt.Errorf("no such credential: %s", name)
+}
+
+func testGetCredential(t *testing.T, cp CredentialProvider, tc *testCase) {
+	actual, actualErr := cp.GetCredential(tc.name)
+	if !reflect.DeepEqual(actual, tc.exp) {
+		t.Fatalf("test case %s failed: want: %#v, have: %#v", tc.name, tc.exp, actual)
+	}
+
+	if !reflect.DeepEqual(actualErr, tc.expErr) {
+		t.Fatalf("test case %s failed: want: %s, have: %s", tc.name, tc.expErr, actualErr)
+	}
+}
+
+func verifySetAndGetCredential(t *testing.T, cp CredentialProvider, tc *testCase) {
+	err := cp.SetCredential(tc.name, tc.exp)
+	if err != nil {
+		t.Fatalf("test case %s failed: cannot set credential: %v", tc.name, err)
+	}
+
+	testGetCredential(t, cp, tc)
+}
+
+func TestNotExist(t *testing.T) {
+	cp := NewInmemCredentialProvider()
+	tc := &testCase{"nonexistent", nil, createMissingError("nonexistent")}
+	testGetCredential(t, cp, tc)
+}
+
+func TestSetAndGetApiToken(t *testing.T) {
+	cp := NewInmemCredentialProvider()
+	tc := &testCase{"testcredential", &RepoCredential{APIToken: "some token here"}, nil}
+	verifySetAndGetCredential(t, cp, tc)
+}
+
+func TestSetAndGetBasicAuth(t *testing.T) {
+	cp := NewInmemCredentialProvider()
+	ba := BasicAuthCredential{Username: "user", Password: "pass"}
+	tc := &testCase{"testcredential", &RepoCredential{BasicAuth: ba}, nil}
+	verifySetAndGetCredential(t, cp, tc)
+}

pkg/repo/inmem_repo_service.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package repo
+
+import (
+	"fmt"
+	"strings"
+)
+
+type inmemRepoService struct {
+	repositories map[string]Repo
+}
+
+// NewInmemRepoService returns a new memory based repository service.
+func NewInmemRepoService() RepoService {
+	rs := &inmemRepoService{
+		repositories: make(map[string]Repo),
+	}
+
+	r, err := NewPublicGCSRepo(nil)
+	if err == nil {
+		rs.Create(r)
+	}
+
+	return rs
+}
+
+// List returns the list of all known chart repositories
+func (rs *inmemRepoService) List() ([]Repo, error) {
+	ret := []Repo{}
+	for _, r := range rs.repositories {
+		ret = append(ret, r)
+	}
+
+	return ret, nil
+}
+
+// Create adds a known repository to the list
+func (rs *inmemRepoService) Create(repository Repo) error {
+	rs.repositories[repository.GetName()] = repository
+	return nil
+}
+
+// Get returns the repository with the given name
+func (rs *inmemRepoService) Get(name string) (Repo, error) {
+	r, ok := rs.repositories[name]
+	if !ok {
+		return nil, fmt.Errorf("Failed to find repository named %s", name)
+	}
+
+	return r, nil
+}
+
+// GetByURL returns the repository that backs the given URL
+func (rs *inmemRepoService) GetByURL(URL string) (Repo, error) {
+	var found Repo
+	for _, r := range rs.repositories {
+		rURL := r.GetURL()
+		if strings.HasPrefix(URL, rURL) {
+			if found == nil || len(found.GetURL()) < len(rURL) {
+				found = r
+			}
+		}
+	}
+
+	if found == nil {
+		return nil, fmt.Errorf("Failed to find repository for url: %s", URL)
+	}
+
+	return found, nil
+}
+
+// Delete removes a known repository from the list
+func (rs *inmemRepoService) Delete(name string) error {
+	_, ok := rs.repositories[name]
+	if !ok {
+		return fmt.Errorf("Failed to find repository named %s", name)
+	}
+
+	delete(rs.repositories, name)
+	return nil
+}

pkg/repo/inmem_repo_service_test.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package repo
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestService(t *testing.T) {
+	rs := NewInmemRepoService()
+	repos, err := rs.List()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(repos) != 1 {
+		t.Fatalf("unexpected repo count; want: %d, have %d.", 1, len(repos))
+	}
+
+	tr := repos[0]
+	if err := validateRepo(tr, GCSPublicRepoName, GCSPublicRepoURL, "", GCSRepoFormat, GCSRepoType); err != nil {
+		t.Fatal(err)
+	}
+
+	r1, err := rs.Get(GCSPublicRepoName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(r1, tr) {
+		t.Fatalf("invalid repo returned; want: %#v, have %#v.", tr, r1)
+	}
+
+	r2, err := rs.GetByURL(GCSPublicRepoURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(r2, tr) {
+		t.Fatalf("invalid repo returned; want: %#v, have %#v.", tr, r2)
+	}
+
+	if err := rs.Delete(GCSPublicRepoName); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := rs.Get(GCSPublicRepoName); err == nil {
+		t.Fatalf("deleted repo named %s returned", GCSPublicRepoName)
+	}
+}
+
+func TestGetRepoWithInvalidName(t *testing.T) {
+	invalidName := "InvalidRepoName"
+	rs := NewInmemRepoService()
+	_, err := rs.Get(invalidName)
+	if err == nil {
+		t.Fatalf("found repo with invalid name: %s", invalidName)
+	}
+}
+
+func TestGetRepoWithInvalidURL(t *testing.T) {
+	invalidURL := "https://not.a.valid/url"
+	rs := NewInmemRepoService()
+	_, err := rs.GetByURL(invalidURL)
+	if err == nil {
+		t.Fatalf("found repo with invalid URL: %s", invalidURL)
+	}
+}
+
+func TestDeleteRepoWithInvalidName(t *testing.T) {
+	invalidName := "InvalidRepoName"
+	rs := NewInmemRepoService()
+	err := rs.Delete(invalidName)
+	if err == nil {
+		t.Fatalf("deleted repo with invalid name: %s", invalidName)
+	}
+}

pkg/repo/repoprovider.go

@@ -0,0 +1,257 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package repo
+
+import (
+	"github.com/kubernetes/helm/pkg/chart"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	storage "google.golang.org/api/storage/v1"
+
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"sync"
+)
+
+// RepoProvider is a factory for ChartRepo instances.
+type RepoProvider interface {
+	GetRepoByURL(URL string) (ChartRepo, error)
+	GetRepoByName(repoName string) (ChartRepo, error)
+	GetChartByReference(reference string) (*chart.Chart, error)
+}
+
+type repoProvider struct {
+	sync.RWMutex
+	rs    RepoService
+	cp    CredentialProvider
+	gcsrp GCSRepoProvider
+	repos map[string]ChartRepo
+}
+
+// NewRepoProvider creates a new repository provider.
+func NewRepoProvider(rs RepoService, gcsrp GCSRepoProvider, cp CredentialProvider) RepoProvider {
+	return newRepoProvider(rs, gcsrp, cp)
+}
+
+// newRepoProvider creates a new repository provider.
+func newRepoProvider(rs RepoService, gcsrp GCSRepoProvider, cp CredentialProvider) *repoProvider {
+	if rs == nil {
+		rs = NewInmemRepoService()
+	}
+
+	if cp == nil {
+		cp = NewInmemCredentialProvider()
+	}
+
+	if gcsrp == nil {
+		gcsrp = NewGCSRepoProvider(cp)
+	}
+
+	repos := make(map[string]ChartRepo)
+	rp := &repoProvider{rs: rs, gcsrp: gcsrp, cp: cp, repos: repos}
+	return rp
+}
+
+// GetRepoService returns the repository service used by this repository provider.
+func (rp *repoProvider) GetRepoService() RepoService {
+	return rp.rs
+}
+
+// GetCredentialProvider returns the credential provider used by this repository provider.
+func (rp *repoProvider) GetCredentialProvider() CredentialProvider {
+	return rp.cp
+}
+
+// GetGCSRepoProvider returns the GCS repository provider used by this repository provider.
+func (rp *repoProvider) GetGCSRepoProvider() GCSRepoProvider {
+	return rp.gcsrp
+}
+
+// GetRepoByName returns the repository with the given name.
+func (rp *repoProvider) GetRepoByName(repoName string) (ChartRepo, error) {
+	rp.Lock()
+	defer rp.Unlock()
+
+	if r, ok := rp.repos[repoName]; ok {
+		return r, nil
+	}
+
+	cr, err := rp.rs.Get(repoName)
+	if err != nil {
+		return nil, err
+	}
+
+	return rp.createRepoByType(cr)
+}
+
+func (rp *repoProvider) createRepoByType(r Repo) (ChartRepo, error) {
+	switch r.GetType() {
+	case GCSRepoType:
+		cr, err := rp.gcsrp.GetGCSRepo(r)
+		if err != nil {
+			return nil, err
+		}
+
+		return rp.createRepo(cr)
+	}
+
+	return nil, fmt.Errorf("unknown repository type: %s", r.GetType())
+}
+
+func (rp *repoProvider) createRepo(cr ChartRepo) (ChartRepo, error) {
+	name := cr.GetName()
+	if _, ok := rp.repos[name]; ok {
+		return nil, fmt.Errorf("respository named %s already exists", name)
+	}
+
+	rp.repos[name] = cr
+	return cr, nil
+}
+
+// GetRepoByURL returns the repository whose URL is a prefix of the given URL.
+func (rp *repoProvider) GetRepoByURL(URL string) (ChartRepo, error) {
+	rp.Lock()
+	defer rp.Unlock()
+
+	if r := rp.findRepoByURL(URL); r != nil {
+		return r, nil
+	}
+
+	cr, err := rp.rs.GetByURL(URL)
+	if err != nil {
+		return nil, err
+	}
+
+	return rp.createRepoByType(cr)
+}
+
+func (rp *repoProvider) findRepoByURL(URL string) ChartRepo {
+	var found ChartRepo
+	for _, r := range rp.repos {
+		rURL := r.GetURL()
+		if strings.HasPrefix(URL, rURL) {
+			if found == nil || len(found.GetURL()) < len(rURL) {
+				found = r
+			}
+		}
+	}
+
+	return found
+}
+
+// GetChartByReference maps the supplied chart reference into a fully qualified
+// URL, uses the URL to find the repository it references, queries the repository
+// for the chart by URL, and returns the result.
+func (rp *repoProvider) GetChartByReference(reference string) (*chart.Chart, error) {
+	l, err := ParseGCSChartReference(reference)
+	if err != nil {
+		return nil, err
+	}
+
+	URL, err := l.Long(true)
+	if err != nil {
+		return nil, fmt.Errorf("invalid reference %s: %s", reference, err)
+	}
+
+	r, err := rp.GetRepoByURL(URL)
+	if err != nil {
+		return nil, err
+	}
+
+	name := fmt.Sprintf("%s-%s.tgz", l.Name, l.Version)
+	return r.GetChart(name)
+}
+
+// GCSRepoProvider is a factory for GCS Repo instances.
+type GCSRepoProvider interface {
+	GetGCSRepo(r Repo) (ObjectStorageRepo, error)
+}
+
+type gcsRepoProvider struct {
+	cp CredentialProvider
+}
+
+// NewGCSRepoProvider creates a GCSRepoProvider.
+func NewGCSRepoProvider(cp CredentialProvider) GCSRepoProvider {
+	if cp == nil {
+		cp = NewInmemCredentialProvider()
+	}
+
+	return gcsRepoProvider{cp: cp}
+}
+
+// GetGCSRepo returns a new Google Cloud Storage repository. If a credential is specified, it will try to
+// fetch it and use it, and if the credential isn't found, it will fall back to an unauthenticated client.
+func (gcsrp gcsRepoProvider) GetGCSRepo(r Repo) (ObjectStorageRepo, error) {
+	client, err := gcsrp.createGCSClient(r.GetCredentialName())
+	if err != nil {
+		return nil, err
+	}
+
+	return NewGCSRepo(r.GetName(), r.GetURL(), r.GetCredentialName(), client)
+}
+
+func (gcsrp gcsRepoProvider) createGCSClient(credentialName string) (*http.Client, error) {
+	if credentialName == "" {
+		return http.DefaultClient, nil
+	}
+
+	c, err := gcsrp.cp.GetCredential(credentialName)
+	if err != nil {
+		log.Printf("credential named %s not found: %s", credentialName, err)
+		log.Print("falling back to the default client")
+		return http.DefaultClient, nil
+	}
+
+	config, err := google.JWTConfigFromJSON([]byte(c.ServiceAccount), storage.DevstorageReadOnlyScope)
+	if err != nil {
+		log.Fatalf("cannot parse client secret file: %s", err)
+	}
+
+	return config.Client(oauth2.NoContext), nil
+}
+
+// IsGCSChartReference returns true if the supplied string is a reference to a chart in a GCS repository
+func IsGCSChartReference(r string) bool {
+	if _, err := ParseGCSChartReference(r); err != nil {
+		return false
+	}
+
+	return true
+}
+
+// ParseGCSChartReference parses a reference to a chart in a GCS repository and returns the URL for the chart
+func ParseGCSChartReference(r string) (*chart.Locator, error) {
+	l, err := chart.Parse(r)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse chart reference %s: %s", r, err)
+	}
+
+	URL, err := l.Long(true)
+	if err != nil {
+		return nil, fmt.Errorf("chart reference %s does not resolve to a URL: %s", r, err)
+	}
+
+	m := GCSChartURLMatcher.FindStringSubmatch(URL)
+	if len(m) != 4 {
+		return nil, fmt.Errorf("chart reference %s resolve to invalid URL: %s", r, URL)
+	}
+
+	return l, nil
+}

pkg/repo/repoprovider_test.go

@@ -0,0 +1,170 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package repo
+
+import (
+	"github.com/kubernetes/helm/pkg/chart"
+
+	"reflect"
+	"testing"
+)
+
+var (
+	TestShortReference = "helm:gs/" + TestRepoBucket + "/" + TestChartName + "#" + TestChartVersion
+	TestLongReference  = TestRepoURL + "/" + TestArchiveName
+)
+
+var ValidChartReferences = []string{
+	TestShortReference,
+	TestLongReference,
+}
+
+var InvalidChartReferences = []string{
+	"gs://missing-chart-segment",
+	"https://not-a-gcs-url",
+	"file://local-chart-reference",
+}
+
+func TestRepoProvider(t *testing.T) {
+	rp := NewRepoProvider(nil, nil, nil)
+	haveRepo, err := rp.GetRepoByName(GCSPublicRepoName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := validateRepo(haveRepo, GCSPublicRepoName, GCSPublicRepoURL, "", GCSRepoFormat, GCSRepoType); err != nil {
+		t.Fatal(err)
+	}
+
+	castRepo, ok := haveRepo.(ObjectStorageRepo)
+	if !ok {
+		t.Fatalf("invalid repo type, want: ObjectStorageRepo, have: %T.", haveRepo)
+	}
+
+	wantBucket := GCSPublicRepoBucket
+	haveBucket := castRepo.GetBucket()
+	if haveBucket != wantBucket {
+		t.Fatalf("unexpected bucket; want: %s, have %s.", wantBucket, haveBucket)
+	}
+
+	wantRepo := haveRepo
+	haveRepo, err = rp.GetRepoByURL(GCSPublicRepoURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(wantRepo, haveRepo) {
+		t.Fatalf("retrieved invalid repo; want: %#v, have %#v.", haveRepo, wantRepo)
+	}
+}
+
+func TestGetRepoByNameWithInvalidName(t *testing.T) {
+	var invalidName = "InvalidRepoName"
+	rp := NewRepoProvider(nil, nil, nil)
+	_, err := rp.GetRepoByName(invalidName)
+	if err == nil {
+		t.Fatalf("found repo using invalid name: %s", invalidName)
+	}
+}
+
+func TestGetRepoByURLWithInvalidURL(t *testing.T) {
+	var invalidURL = "https://valid.url/wrong/scheme"
+	rp := NewRepoProvider(nil, nil, nil)
+	_, err := rp.GetRepoByURL(invalidURL)
+	if err == nil {
+		t.Fatalf("found repo using invalid URL: %s", invalidURL)
+	}
+}
+
+func TestGetChartByReferenceWithValidReferences(t *testing.T) {
+	rp := getTestRepoProvider(t)
+	wantFile, err := chart.LoadChartfile(TestChartFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, vcr := range ValidChartReferences {
+		t.Logf("getting chart by reference: %s", vcr)
+		tc, err := rp.GetChartByReference(vcr)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		haveFile := tc.Chartfile()
+		if reflect.DeepEqual(wantFile, haveFile) {
+			t.Fatalf("retrieved invalid chart\nwant:%#v\nhave:\n%#v\n", wantFile, haveFile)
+		}
+	}
+}
+
+func getTestRepoProvider(t *testing.T) RepoProvider {
+	rp := newRepoProvider(nil, nil, nil)
+	rs := rp.GetRepoService()
+	tr, err := newRepo(TestRepoName, TestRepoURL, TestRepoCredentialName, TestRepoFormat, TestRepoType)
+	if err != nil {
+		t.Fatalf("cannot create test repository: %s", err)
+	}
+
+	if err := rs.Create(tr); err != nil {
+		t.Fatalf("cannot initialize repository service: %s", err)
+	}
+
+	return rp
+}
+
+func TestGetChartByReferenceWithInvalidReferences(t *testing.T) {
+	rp := NewRepoProvider(nil, nil, nil)
+	for _, icr := range InvalidChartReferences {
+		_, err := rp.GetChartByReference(icr)
+		if err == nil {
+			t.Fatalf("found chart using invalid reference: %s", icr)
+		}
+	}
+}
+
+func TestIsGCSChartReferenceWithValidReferences(t *testing.T) {
+	for _, vcr := range ValidChartReferences {
+		if !IsGCSChartReference(vcr) {
+			t.Fatalf("valid chart reference %s not accepted", vcr)
+		}
+	}
+}
+
+func TestIsGCSChartReferenceWithInvalidReferences(t *testing.T) {
+	for _, icr := range InvalidChartReferences {
+		if IsGCSChartReference(icr) {
+			t.Fatalf("invalid chart reference %s accepted", icr)
+		}
+	}
+}
+
+func TestParseGCSChartReferences(t *testing.T) {
+	for _, vcr := range ValidChartReferences {
+		if _, err := ParseGCSChartReference(vcr); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestParseGCSChartReferenceWithInvalidReferences(t *testing.T) {
+	for _, icr := range InvalidChartReferences {
+		if _, err := ParseGCSChartReference(icr); err == nil {
+			t.Fatalf("invalid chart reference %s parsed correctly", icr)
+		}
+	}
+}