msb-public
/
helm
mirror of https://github.com/helm/helm
1
0
Code Issues Projects Releases Wiki Activity

Add new GCS repository

Browse Source
pull/408/head
jackgr 9 years ago
parent 9d8567e07a
commit d0506d403b
2 changed files with 145 additions and 133 deletions
Show Stats Download Patch File Download Diff File

120
pkg/repo/gcs_repo.go
Unescape View File

@ -18,7 +18,6 @@ package repo
import ( import (
"github.com/kubernetes/helm/pkg/chart" "github.com/kubernetes/helm/pkg/chart"
"github.com/kubernetes/helm/pkg/common"
"github.com/kubernetes/helm/pkg/util" "github.com/kubernetes/helm/pkg/util"
storage "google.golang.org/api/storage/v1" storage "google.golang.org/api/storage/v1"
@ -27,34 +26,64 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"regexp" "regexp"
"strings"
) )
// GCSRepo implements the ObjectStorageRepo interface // GCSRepoURLMatcher matches the GCS repository URL format (gs://<bucket>).
// for Google Cloud Storage. var GCSRepoURLMatcher = regexp.MustCompile("gs://(.*)")
//
// A GCSRepo root must be a directory that contains all the available charts. // GCSChartURLMatcher matches the GCS chart URL format (gs://<bucket>/<name>-<version>.tgz).
type GCSRepo struct { var GCSChartURLMatcher = regexp.MustCompile("gs://(.*)/(.*)-(.*).tgz")
chartRepo // A GCSRepo is a chartRepo
bucket string const (
credentialName string // GCSRepoType identifies the GCS repository type.
httpClient *http.Client GCSRepoType = RepoType("gcs")
service *storage.Service
// GCSRepoFormat identifies the GCS repository format.
// In a GCS repository all charts appear at the top level.
GCSRepoFormat = FlatRepoFormat
// GCSPublicRepoName is the name of the public GCS repository.
GCSPublicRepoName = "kubernetes-charts"
// GCSPublicRepoName is the URL for the public GCS repository.
GCSPublicRepoURL = "gs://" + GCSPublicRepoName
// GCSPublicRepoBucket is the name of the public GCS repository bucket.
GCSPublicRepoBucket = GCSPublicRepoName
)
// gcsRepo implements the ObjectStorageRepo interface for Google Cloud Storage.
type gcsRepo struct {
repo
bucket string
httpClient *http.Client
service *storage.Service
} }
// URLFormatMatcher matches the GCS URL format (gs:). // NewPublicGCSRepo creates a new an ObjectStorageRepo for the public GCS repository.
var URLFormatMatcher = regexp.MustCompile("gs://(.*)") func NewPublicGCSRepo(httpClient *http.Client) (ObjectStorageRepo, error) {
return NewGCSRepo(GCSPublicRepoName, GCSPublicRepoURL, "", nil)
}
var GCSRepoFormat = common.RepoFormat(fmt.Sprintf("%s;%s", common.UnversionedRepo, common.OneLevelRepo)) // NewGCSRepo creates a new ObjectStorageRepo for a given GCS repository.
func NewGCSRepo(name, URL, credentialName string, httpClient *http.Client) (ObjectStorageRepo, error) {
r, err := newRepo(name, URL, credentialName, GCSRepoFormat, GCSRepoType)
if err != nil {
return nil, err
}
// NewGCSRepo creates a GCS repository. return newGCSRepo(r, httpClient)
func NewGCSRepo(name, URL string, httpClient *http.Client) (*GCSRepo, error) { }
m := URLFormatMatcher.FindStringSubmatch(URL)
func newGCSRepo(r *repo, httpClient *http.Client) (*gcsRepo, error) {
URL := r.GetURL()
m := GCSRepoURLMatcher.FindStringSubmatch(URL)
if len(m) != 2 { if len(m) != 2 {
return nil, fmt.Errorf("URL must be of the form gs://<bucket>, was %s", URL) return nil, fmt.Errorf("URL must be of the form gs://<bucket>, was %s", URL)
} }
cr, err := newRepo(name, URL, string(GCSRepoFormat), string(common.GCSRepoType)) if err := validateRepoType(r.GetType()); err != nil {
if err != nil {
return nil, err return nil, err
} }
@ -62,30 +91,33 @@ func NewGCSRepo(name, URL string, httpClient *http.Client) (*GCSRepo, error) {
httpClient = http.DefaultClient httpClient = http.DefaultClient
} }
gs, err := storage.New(httpClient) gcs, err := storage.New(httpClient)
if err != nil { if err != nil {
return nil, fmt.Errorf("cannot create storage service for %s: %s", URL, err) return nil, fmt.Errorf("cannot create storage service for %s: %s", URL, err)
} }
result := &GCSRepo{ gcsr := &gcsRepo{
chartRepo: *cr, repo: *r,
httpClient: httpClient, httpClient: httpClient,
service: gs, service: gcs,
bucket: m[1], bucket: m[1],
} }
return result, nil return gcsr, nil
} }
// GetBucket returns the repository bucket. func validateRepoType(repoType RepoType) error {
func (g *GCSRepo) GetBucket() string { switch repoType {
return g.bucket case GCSRepoType:
return nil
}
return fmt.Errorf("unknown repository type: %s", repoType)
} }
// ListCharts lists charts in this chart repository whose string values conform to the // ListCharts lists charts in this chart repository whose string values conform to the
// supplied regular expression, or all charts, if the regular expression is nil. // supplied regular expression, or all charts, if the regular expression is nil.
func (g *GCSRepo) ListCharts(regex *regexp.Regexp) ([]string, error) { func (g *gcsRepo) ListCharts(regex *regexp.Regexp) ([]string, error) {
// List all files in the bucket/prefix that contain the
charts := []string{} charts := []string{}
// List all objects in a bucket using pagination // List all objects in a bucket using pagination
@ -96,12 +128,14 @@ func (g *GCSRepo) ListCharts(regex *regexp.Regexp) ([]string, error) {
if pageToken != "" { if pageToken != "" {
call = call.PageToken(pageToken) call = call.PageToken(pageToken)
} }
res, err := call.Do() res, err := call.Do()
if err != nil { if err != nil {
return nil, err return nil, err
} }
for _, object := range res.Items { for _, object := range res.Items {
// Charts should be named bucket/chart-X.Y.Z.tgz, so tease apart the version here // Charts should be named bucket/chart-X.Y.Z.tgz, so tease apart the name
m := ChartNameMatcher.FindStringSubmatch(object.Name) m := ChartNameMatcher.FindStringSubmatch(object.Name)
if len(m) != 3 { if len(m) != 3 {
continue continue
@ -121,8 +155,8 @@ func (g *GCSRepo) ListCharts(regex *regexp.Regexp) ([]string, error) {
} }
// GetChart retrieves, unpacks and returns a chart by name. // GetChart retrieves, unpacks and returns a chart by name.
func (g *GCSRepo) GetChart(name string) (*chart.Chart, error) { func (g *gcsRepo) GetChart(name string) (*chart.Chart, error) {
// Charts should be named bucket/chart-X.Y.Z.tgz, so tease apart the version here // Charts should be named bucket/chart-X.Y.Z.tgz, so check that the name matches
if !ChartNameMatcher.MatchString(name) { if !ChartNameMatcher.MatchString(name) {
return nil, fmt.Errorf("name must be of the form <name>-<version>.tgz, was %s", name) return nil, fmt.Errorf("name must be of the form <name>-<version>.tgz, was %s", name)
} }
@ -130,33 +164,31 @@ func (g *GCSRepo) GetChart(name string) (*chart.Chart, error) {
call := g.service.Objects.Get(g.bucket, name) call := g.service.Objects.Get(g.bucket, name)
object, err := call.Do() object, err := call.Do()
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("cannot get storage object named %s/%s: %s", g.bucket, name, err)
} }
u, err := url.Parse(object.MediaLink) u, err := url.Parse(object.MediaLink)
if err != nil { if err != nil {
return nil, fmt.Errorf("Cannot parse URL %s for chart %s/%s: %s", return nil, fmt.Errorf("cannot parse URL %s for chart %s/%s: %s",
object.MediaLink, object.Bucket, object.Name, err) object.MediaLink, object.Bucket, object.Name, err)
} }
getter := util.NewHTTPClient(3, g.httpClient, util.NewSleeper()) getter := util.NewHTTPClient(3, g.httpClient, util.NewSleeper())
body, code, err := getter.Get(u.String()) body, code, err := getter.Get(u.String())
if err != nil { if err != nil {
return nil, fmt.Errorf("Cannot fetch URL %s for chart %s/%s: %d %s", return nil, fmt.Errorf("cannot fetch URL %s for chart %s/%s: %d %s",
object.MediaLink, object.Bucket, object.Name, code, err) object.MediaLink, object.Bucket, object.Name, code, err)
} }
return chart.Load(body) return chart.LoadDataFromReader(strings.NewReader(body))
} }
// Do performs an HTTP operation on the receiver's httpClient. // GetBucket returns the repository bucket.
func (g *GCSRepo) Do(req *http.Request) (resp *http.Response, err error) { func (g *gcsRepo) GetBucket() string {
return g.httpClient.Do(req) return g.bucket
} }
// TODO: Remove GetShortURL when no longer needed. // Do performs an HTTP operation on the receiver's httpClient.
func (g *gcsRepo) Do(req *http.Request) (resp *http.Response, err error) {
// GetShortURL returns the URL without the scheme. return g.httpClient.Do(req)
func (g GCSRepo) GetShortURL() string {
return util.TrimURLScheme(g.URL)
} }

158
pkg/repo/gcs_repo_test.go
Unescape View File

@ -18,7 +18,6 @@ package repo
import ( import (
"github.com/kubernetes/helm/pkg/chart" "github.com/kubernetes/helm/pkg/chart"
"github.com/kubernetes/helm/pkg/common"
"os" "os"
"reflect" "reflect"
@ -27,129 +26,110 @@ import (
) )
var ( var (
TestArchiveBucket = os.Getenv("TEST_ARCHIVE_BUCKET") TestArchiveURL = os.Getenv("TEST_ARCHIVE_URL")
TestArchiveName = "frobnitz-0.0.1.tgz" TestChartName = "frobnitz"
TestChartVersion = "0.0.1"
TestArchiveName = TestChartName + "-" + TestChartVersion + ".tgz"
TestChartFile = "testdata/frobnitz/Chart.yaml" TestChartFile = "testdata/frobnitz/Chart.yaml"
TestShouldFindRegex = regexp.MustCompile(TestArchiveName) TestShouldFindRegex = regexp.MustCompile(TestArchiveName)
TestShouldNotFindRegex = regexp.MustCompile("foobar") TestShouldNotFindRegex = regexp.MustCompile("foobar")
) )
func TestValidGSURL(t *testing.T) { func TestValidGSURL(t *testing.T) {
var validURL = "gs://bucket" tr := getTestRepo(t)
tr, err := NewGCSRepo("testName", validURL, nil) err := validateRepo(tr, TestRepoName, TestRepoURL, TestRepoCredentialName, TestRepoFormat, TestRepoType)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
wantType := common.GCSRepoType wantBucket := TestRepoBucket
haveType := tr.GetRepoType() haveBucket := tr.GetBucket()
if haveType != wantType { if haveBucket != wantBucket {
t.Fatalf("unexpected repo type; want: %s, have %s.", wantType, haveType) t.Fatalf("unexpected bucket; want: %s, have %s.", wantBucket, haveBucket)
} }
wantFormat := GCSRepoFormat
haveFormat := tr.GetRepoFormat()
if haveFormat != wantFormat {
t.Fatalf("unexpected repo format; want: %s, have %s.", wantFormat, haveFormat)
}
} }
func TestInvalidGSURL(t *testing.T) { func TestInvalidGSURL(t *testing.T) {
var invalidURL = "https://bucket" var invalidGSURL = "https://valid.url/wrong/scheme"
_, err := NewGCSRepo("testName", invalidURL, nil) _, err := NewGCSRepo(TestRepoName, invalidGSURL, TestRepoCredentialName, nil)
if err == nil { if err == nil {
t.Fatalf("expected error did not occur for invalid URL") t.Fatalf("expected error did not occur for invalid GS URL")
} }
} }
func TestListCharts(t *testing.T) { func TestListCharts(t *testing.T) {
if TestArchiveBucket != "" { tr := getTestRepo(t)
tr, err := NewGCSRepo("testName", TestArchiveBucket, nil) charts, err := tr.ListCharts(nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
charts, err := tr.ListCharts(nil) if len(charts) != 1 {
if err != nil { t.Fatalf("expected one chart in list, got %d", len(charts))
t.Fatal(err) }
}
haveName := charts[0]
if len(charts) != 1 { wantName := TestArchiveName
t.Fatalf("expected one chart in test bucket, got %d", len(charts)) if haveName != wantName {
} t.Fatalf("expected chart named %s, got %s", wantName, haveName)
name := charts[0]
if name != TestArchiveName {
t.Fatalf("expected chart named %s in test bucket, got %s", TestArchiveName, name)
}
} }
} }
func TestListChartsWithShouldFindRegex(t *testing.T) { func TestListChartsWithShouldFindRegex(t *testing.T) {
if TestArchiveBucket != "" { tr := getTestRepo(t)
tr, err := NewGCSRepo("testName", TestArchiveBucket, nil) charts, err := tr.ListCharts(TestShouldFindRegex)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
charts, err := tr.ListCharts(TestShouldFindRegex) if len(charts) != 1 {
if err != nil { t.Fatalf("expected one chart to match regex, got %d", len(charts))
t.Fatal(err)
}
if len(charts) != 1 {
t.Fatalf("expected one chart to match regex, got %d", len(charts))
}
} }
} }
func TestListChartsWithShouldNotFindRegex(t *testing.T) { func TestListChartsWithShouldNotFindRegex(t *testing.T) {
if TestArchiveBucket != "" { tr := getTestRepo(t)
tr, err := NewGCSRepo("testName", TestArchiveBucket, nil) charts, err := tr.ListCharts(TestShouldNotFindRegex)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
charts, err := tr.ListCharts(TestShouldNotFindRegex) if len(charts) != 0 {
if err != nil { t.Fatalf("expected zero charts to match regex, got %d", len(charts))
t.Fatal(err)
}
if len(charts) != 0 {
t.Fatalf("expected zero charts to match regex, got %d", len(charts))
}
} }
} }
func TestGetChart(t *testing.T) { func TestGetChart(t *testing.T) {
if TestArchiveBucket != "" { tr := getTestRepo(t)
tr, err := NewGCSRepo("testName", TestArchiveBucket, nil) tc, err := tr.GetChart(TestArchiveName)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
tc, err := tr.GetChart(TestArchiveName) haveFile := tc.Chartfile()
if err != nil { wantFile, err := chart.LoadChartfile(TestChartFile)
t.Fatal(err) if err != nil {
} t.Fatal(err)
}
have := tc.Chartfile()
want, err := chart.LoadChartfile(TestChartFile) if reflect.DeepEqual(wantFile, haveFile) {
if err != nil { t.Fatalf("retrieved invalid chart\nwant:%#v\nhave:\n%#v\n", wantFile, haveFile)
t.Fatal(err)
}
if reflect.DeepEqual(want, have) {
t.Fatalf("retrieved an invalid chart\nwant:%#v\nhave:\n%#v\n", want, have)
}
} }
} }
func TestGetChartWithInvalidName(t *testing.T) { func TestGetChartWithInvalidName(t *testing.T) {
var invalidURL = "https://bucket" tr := getTestRepo(t)
_, err := NewGCSRepo("testName", invalidURL, nil) _, err := tr.GetChart("NotAValidArchiveName")
if err == nil { if err == nil {
t.Fatalf("expected error did not occur for invalid URL") t.Fatalf("found chart using invalid archive name")
} }
} }
func getTestRepo(t *testing.T) ObjectStorageRepo {
tr, err := NewGCSRepo(TestRepoName, TestRepoURL, TestRepoCredentialName, nil)
if err != nil {
t.Fatal(err)
}
return tr
}

Write Preview
Loading…
Cancel
Save