msb-public
/
helm
mirror of https://github.com/helm/helm
1
0
Code Issues Projects Releases Wiki Activity

Add new GCS repository

Browse Source
pull/408/head
jackgr 9 years ago
parent 9d8567e07a
commit d0506d403b
2 changed files with 145 additions and 133 deletions
Show Stats Download Patch File Download Diff File

120
pkg/repo/gcs_repo.go
Unescape View File

@ -18,7 +18,6 @@ package repo
import (
"github.com/kubernetes/helm/pkg/chart"
"github.com/kubernetes/helm/pkg/common"
"github.com/kubernetes/helm/pkg/util"
storage "google.golang.org/api/storage/v1"
@ -27,34 +26,64 @@ import (
"net/http"
"net/url"
"regexp"
"strings"
)
// GCSRepo implements the ObjectStorageRepo interface
// for Google Cloud Storage.
//
// A GCSRepo root must be a directory that contains all the available charts.
type GCSRepo struct {
chartRepo // A GCSRepo is a chartRepo
bucket string
credentialName string
httpClient *http.Client
service *storage.Service
// GCSRepoURLMatcher matches the GCS repository URL format (gs://<bucket>).
var GCSRepoURLMatcher = regexp.MustCompile("gs://(.*)")
// GCSChartURLMatcher matches the GCS chart URL format (gs://<bucket>/<name>-<version>.tgz).
var GCSChartURLMatcher = regexp.MustCompile("gs://(.*)/(.*)-(.*).tgz")
const (
// GCSRepoType identifies the GCS repository type.
GCSRepoType = RepoType("gcs")
// GCSRepoFormat identifies the GCS repository format.
// In a GCS repository all charts appear at the top level.
GCSRepoFormat = FlatRepoFormat
// GCSPublicRepoName is the name of the public GCS repository.
GCSPublicRepoName = "kubernetes-charts"
// GCSPublicRepoName is the URL for the public GCS repository.
GCSPublicRepoURL = "gs://" + GCSPublicRepoName
// GCSPublicRepoBucket is the name of the public GCS repository bucket.
GCSPublicRepoBucket = GCSPublicRepoName
)
// gcsRepo implements the ObjectStorageRepo interface for Google Cloud Storage.
type gcsRepo struct {
repo
bucket string
httpClient *http.Client
service *storage.Service
}
// URLFormatMatcher matches the GCS URL format (gs:).
var URLFormatMatcher = regexp.MustCompile("gs://(.*)")
// NewPublicGCSRepo creates a new an ObjectStorageRepo for the public GCS repository.
func NewPublicGCSRepo(httpClient *http.Client) (ObjectStorageRepo, error) {
return NewGCSRepo(GCSPublicRepoName, GCSPublicRepoURL, "", nil)
}
var GCSRepoFormat = common.RepoFormat(fmt.Sprintf("%s;%s", common.UnversionedRepo, common.OneLevelRepo))
// NewGCSRepo creates a new ObjectStorageRepo for a given GCS repository.
func NewGCSRepo(name, URL, credentialName string, httpClient *http.Client) (ObjectStorageRepo, error) {
r, err := newRepo(name, URL, credentialName, GCSRepoFormat, GCSRepoType)
if err != nil {
return nil, err
}
// NewGCSRepo creates a GCS repository.
func NewGCSRepo(name, URL string, httpClient *http.Client) (*GCSRepo, error) {
m := URLFormatMatcher.FindStringSubmatch(URL)
return newGCSRepo(r, httpClient)
}
func newGCSRepo(r *repo, httpClient *http.Client) (*gcsRepo, error) {
URL := r.GetURL()
m := GCSRepoURLMatcher.FindStringSubmatch(URL)
if len(m) != 2 {
return nil, fmt.Errorf("URL must be of the form gs://<bucket>, was %s", URL)
}
cr, err := newRepo(name, URL, string(GCSRepoFormat), string(common.GCSRepoType))
if err != nil {
if err := validateRepoType(r.GetType()); err != nil {
return nil, err
}
@ -62,30 +91,33 @@ func NewGCSRepo(name, URL string, httpClient *http.Client) (*GCSRepo, error) {
httpClient = http.DefaultClient
}
gs, err := storage.New(httpClient)
gcs, err := storage.New(httpClient)
if err != nil {
return nil, fmt.Errorf("cannot create storage service for %s: %s", URL, err)
}
result := &GCSRepo{
chartRepo: *cr,
gcsr := &gcsRepo{
repo: *r,
httpClient: httpClient,
service: gs,
service: gcs,
bucket: m[1],
}
return result, nil
return gcsr, nil
}
// GetBucket returns the repository bucket.
func (g *GCSRepo) GetBucket() string {
return g.bucket
func validateRepoType(repoType RepoType) error {
switch repoType {
case GCSRepoType:
return nil
}
return fmt.Errorf("unknown repository type: %s", repoType)
}
// ListCharts lists charts in this chart repository whose string values conform to the
// supplied regular expression, or all charts, if the regular expression is nil.
func (g *GCSRepo) ListCharts(regex *regexp.Regexp) ([]string, error) {
// List all files in the bucket/prefix that contain the
func (g *gcsRepo) ListCharts(regex *regexp.Regexp) ([]string, error) {
charts := []string{}
// List all objects in a bucket using pagination
@ -96,12 +128,14 @@ func (g *GCSRepo) ListCharts(regex *regexp.Regexp) ([]string, error) {
if pageToken != "" {
call = call.PageToken(pageToken)
}
res, err := call.Do()
if err != nil {
return nil, err
}
for _, object := range res.Items {
// Charts should be named bucket/chart-X.Y.Z.tgz, so tease apart the version here
// Charts should be named bucket/chart-X.Y.Z.tgz, so tease apart the name
m := ChartNameMatcher.FindStringSubmatch(object.Name)
if len(m) != 3 {
continue
@ -121,8 +155,8 @@ func (g *GCSRepo) ListCharts(regex *regexp.Regexp) ([]string, error) {
}
// GetChart retrieves, unpacks and returns a chart by name.
func (g *GCSRepo) GetChart(name string) (*chart.Chart, error) {
// Charts should be named bucket/chart-X.Y.Z.tgz, so tease apart the version here
func (g *gcsRepo) GetChart(name string) (*chart.Chart, error) {
// Charts should be named bucket/chart-X.Y.Z.tgz, so check that the name matches
if !ChartNameMatcher.MatchString(name) {
return nil, fmt.Errorf("name must be of the form <name>-<version>.tgz, was %s", name)
}
@ -130,33 +164,31 @@ func (g *GCSRepo) GetChart(name string) (*chart.Chart, error) {
call := g.service.Objects.Get(g.bucket, name)
object, err := call.Do()
if err != nil {
return nil, err
return nil, fmt.Errorf("cannot get storage object named %s/%s: %s", g.bucket, name, err)
}
u, err := url.Parse(object.MediaLink)
if err != nil {
return nil, fmt.Errorf("Cannot parse URL %s for chart %s/%s: %s",
return nil, fmt.Errorf("cannot parse URL %s for chart %s/%s: %s",
object.MediaLink, object.Bucket, object.Name, err)
}
getter := util.NewHTTPClient(3, g.httpClient, util.NewSleeper())
body, code, err := getter.Get(u.String())
if err != nil {
return nil, fmt.Errorf("Cannot fetch URL %s for chart %s/%s: %d %s",
return nil, fmt.Errorf("cannot fetch URL %s for chart %s/%s: %d %s",
object.MediaLink, object.Bucket, object.Name, code, err)
}
return chart.Load(body)
return chart.LoadDataFromReader(strings.NewReader(body))
}
// Do performs an HTTP operation on the receiver's httpClient.
func (g *GCSRepo) Do(req *http.Request) (resp *http.Response, err error) {
return g.httpClient.Do(req)
// GetBucket returns the repository bucket.
func (g *gcsRepo) GetBucket() string {
return g.bucket
}
// TODO: Remove GetShortURL when no longer needed.
// GetShortURL returns the URL without the scheme.
func (g GCSRepo) GetShortURL() string {
return util.TrimURLScheme(g.URL)
// Do performs an HTTP operation on the receiver's httpClient.
func (g *gcsRepo) Do(req *http.Request) (resp *http.Response, err error) {
return g.httpClient.Do(req)
}

158
pkg/repo/gcs_repo_test.go
Unescape View File

@ -18,7 +18,6 @@ package repo
import (
"github.com/kubernetes/helm/pkg/chart"
"github.com/kubernetes/helm/pkg/common"
"os"
"reflect"
@ -27,129 +26,110 @@ import (
)
var (
TestArchiveBucket = os.Getenv("TEST_ARCHIVE_BUCKET")
TestArchiveName = "frobnitz-0.0.1.tgz"
TestArchiveURL = os.Getenv("TEST_ARCHIVE_URL")
TestChartName = "frobnitz"
TestChartVersion = "0.0.1"
TestArchiveName = TestChartName + "-" + TestChartVersion + ".tgz"
TestChartFile = "testdata/frobnitz/Chart.yaml"
TestShouldFindRegex = regexp.MustCompile(TestArchiveName)
TestShouldNotFindRegex = regexp.MustCompile("foobar")
)
func TestValidGSURL(t *testing.T) {
var validURL = "gs://bucket"
tr, err := NewGCSRepo("testName", validURL, nil)
tr := getTestRepo(t)
err := validateRepo(tr, TestRepoName, TestRepoURL, TestRepoCredentialName, TestRepoFormat, TestRepoType)
if err != nil {
t.Fatal(err)
}
wantType := common.GCSRepoType
haveType := tr.GetRepoType()
if haveType != wantType {
t.Fatalf("unexpected repo type; want: %s, have %s.", wantType, haveType)
wantBucket := TestRepoBucket
haveBucket := tr.GetBucket()
if haveBucket != wantBucket {
t.Fatalf("unexpected bucket; want: %s, have %s.", wantBucket, haveBucket)
}
wantFormat := GCSRepoFormat
haveFormat := tr.GetRepoFormat()
if haveFormat != wantFormat {
t.Fatalf("unexpected repo format; want: %s, have %s.", wantFormat, haveFormat)
}
}
func TestInvalidGSURL(t *testing.T) {
var invalidURL = "https://bucket"
_, err := NewGCSRepo("testName", invalidURL, nil)
var invalidGSURL = "https://valid.url/wrong/scheme"
_, err := NewGCSRepo(TestRepoName, invalidGSURL, TestRepoCredentialName, nil)
if err == nil {
t.Fatalf("expected error did not occur for invalid URL")
t.Fatalf("expected error did not occur for invalid GS URL")
}
}
func TestListCharts(t *testing.T) {
if TestArchiveBucket != "" {
tr, err := NewGCSRepo("testName", TestArchiveBucket, nil)
if err != nil {
t.Fatal(err)
}
charts, err := tr.ListCharts(nil)
if err != nil {
t.Fatal(err)
}
if len(charts) != 1 {
t.Fatalf("expected one chart in test bucket, got %d", len(charts))
}
name := charts[0]
if name != TestArchiveName {
t.Fatalf("expected chart named %s in test bucket, got %s", TestArchiveName, name)
}
tr := getTestRepo(t)
charts, err := tr.ListCharts(nil)
if err != nil {
t.Fatal(err)
}
if len(charts) != 1 {
t.Fatalf("expected one chart in list, got %d", len(charts))
}
haveName := charts[0]
wantName := TestArchiveName
if haveName != wantName {
t.Fatalf("expected chart named %s, got %s", wantName, haveName)
}
}
func TestListChartsWithShouldFindRegex(t *testing.T) {
if TestArchiveBucket != "" {
tr, err := NewGCSRepo("testName", TestArchiveBucket, nil)
if err != nil {
t.Fatal(err)
}
charts, err := tr.ListCharts(TestShouldFindRegex)
if err != nil {
t.Fatal(err)
}
if len(charts) != 1 {
t.Fatalf("expected one chart to match regex, got %d", len(charts))
}
tr := getTestRepo(t)
charts, err := tr.ListCharts(TestShouldFindRegex)
if err != nil {
t.Fatal(err)
}
if len(charts) != 1 {
t.Fatalf("expected one chart to match regex, got %d", len(charts))
}
}
func TestListChartsWithShouldNotFindRegex(t *testing.T) {
if TestArchiveBucket != "" {
tr, err := NewGCSRepo("testName", TestArchiveBucket, nil)
if err != nil {
t.Fatal(err)
}
charts, err := tr.ListCharts(TestShouldNotFindRegex)
if err != nil {
t.Fatal(err)
}
if len(charts) != 0 {
t.Fatalf("expected zero charts to match regex, got %d", len(charts))
}
tr := getTestRepo(t)
charts, err := tr.ListCharts(TestShouldNotFindRegex)
if err != nil {
t.Fatal(err)
}
if len(charts) != 0 {
t.Fatalf("expected zero charts to match regex, got %d", len(charts))
}
}
func TestGetChart(t *testing.T) {
if TestArchiveBucket != "" {
tr, err := NewGCSRepo("testName", TestArchiveBucket, nil)
if err != nil {
t.Fatal(err)
}
tc, err := tr.GetChart(TestArchiveName)
if err != nil {
t.Fatal(err)
}
have := tc.Chartfile()
want, err := chart.LoadChartfile(TestChartFile)
if err != nil {
t.Fatal(err)
}
if reflect.DeepEqual(want, have) {
t.Fatalf("retrieved an invalid chart\nwant:%#v\nhave:\n%#v\n", want, have)
}
tr := getTestRepo(t)
tc, err := tr.GetChart(TestArchiveName)
if err != nil {
t.Fatal(err)
}
haveFile := tc.Chartfile()
wantFile, err := chart.LoadChartfile(TestChartFile)
if err != nil {
t.Fatal(err)
}
if reflect.DeepEqual(wantFile, haveFile) {
t.Fatalf("retrieved invalid chart\nwant:%#v\nhave:\n%#v\n", wantFile, haveFile)
}
}
func TestGetChartWithInvalidName(t *testing.T) {
var invalidURL = "https://bucket"
_, err := NewGCSRepo("testName", invalidURL, nil)
tr := getTestRepo(t)
_, err := tr.GetChart("NotAValidArchiveName")
if err == nil {
t.Fatalf("expected error did not occur for invalid URL")
t.Fatalf("found chart using invalid archive name")
}
}
func getTestRepo(t *testing.T) ObjectStorageRepo {
tr, err := NewGCSRepo(TestRepoName, TestRepoURL, TestRepoCredentialName, nil)
if err != nil {
t.Fatal(err)
}
return tr
}

Write Preview
Loading…
Cancel
Save