From b0e7a43b5bd440ca891293bd3092f18f4d5ea3cf Mon Sep 17 00:00:00 2001 From: Anton Galitsyn Date: Wed, 23 Nov 2016 16:17:23 +0700 Subject: [PATCH 1/7] Construct http.Client for repositories from config, add TLS support --- cmd/helm/dependency_build_test.go | 2 +- cmd/helm/dependency_update_test.go | 2 +- cmd/helm/downloader/chart_downloader.go | 4 +- cmd/helm/downloader/manager.go | 60 ++--- cmd/helm/downloader/manager_test.go | 24 -- cmd/helm/fetch.go | 3 +- cmd/helm/helm_test.go | 8 +- cmd/helm/init.go | 108 ++++++--- cmd/helm/init_test.go | 22 +- cmd/helm/repo_add.go | 72 +++--- cmd/helm/repo_add_test.go | 6 +- cmd/helm/repo_index.go | 4 +- cmd/helm/repo_index_test.go | 4 +- cmd/helm/repo_remove_test.go | 25 ++- cmd/helm/repo_update.go | 36 ++- cmd/helm/repo_update_test.go | 31 ++- cmd/helm/resolver/resolver.go | 2 +- cmd/helm/search.go | 2 +- cmd/helm/search/search.go | 2 +- cmd/helm/search/search_test.go | 4 +- pkg/repo/chartrepo.go | 168 ++++++++++++++ pkg/repo/chartrepo_test.go | 185 +++++++++++++++ pkg/repo/index.go | 285 ++++++++++-------------- pkg/repo/index_test.go | 57 ++--- pkg/repo/local.go | 4 +- pkg/repo/repo.go | 142 ++---------- pkg/repo/repo_test.go | 164 +------------- pkg/repo/repotest/server.go | 21 +- pkg/repo/repotest/server_test.go | 2 +- pkg/tlsutil/tls.go | 68 ++++++ pkg/urlutil/urlutil.go | 66 ++++++ pkg/urlutil/urlutil_test.go | 64 ++++++ 32 files changed, 959 insertions(+), 688 deletions(-) create mode 100644 pkg/repo/chartrepo.go create mode 100644 pkg/repo/chartrepo_test.go create mode 100644 pkg/tlsutil/tls.go create mode 100644 pkg/urlutil/urlutil.go create mode 100644 pkg/urlutil/urlutil_test.go diff --git a/cmd/helm/dependency_build_test.go b/cmd/helm/dependency_build_test.go index 361d3ed6c..9e615a994 100644 --- a/cmd/helm/dependency_build_test.go +++ b/cmd/helm/dependency_build_test.go @@ -103,7 +103,7 @@ func TestDependencyBuildCmd(t *testing.T) { t.Fatal(err) } - i, err := repo.LoadIndexFile(dbc.helmhome.CacheIndex("test")) + i, err := repo.NewChartRepositoryIndexFromFile(dbc.helmhome.CacheIndex("test")) if err != nil { t.Fatal(err) } diff --git a/cmd/helm/dependency_update_test.go b/cmd/helm/dependency_update_test.go index e47e494f5..9c6171fe7 100644 --- a/cmd/helm/dependency_update_test.go +++ b/cmd/helm/dependency_update_test.go @@ -88,7 +88,7 @@ func TestDependencyUpdateCmd(t *testing.T) { t.Fatal(err) } - i, err := repo.LoadIndexFile(duc.helmhome.CacheIndex("test")) + i, err := repo.NewChartRepositoryIndexFromFile(duc.helmhome.CacheIndex("test")) if err != nil { t.Fatal(err) } diff --git a/cmd/helm/downloader/chart_downloader.go b/cmd/helm/downloader/chart_downloader.go index 350ccb6c0..bb64ff7fc 100644 --- a/cmd/helm/downloader/chart_downloader.go +++ b/cmd/helm/downloader/chart_downloader.go @@ -166,7 +166,7 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, er } // Next, we need to load the index, and actually look up the chart. - i, err := repo.LoadIndexFile(c.HelmHome.CacheIndex(repoName)) + i, err := repo.NewChartRepositoryIndexFromFile(c.HelmHome.CacheIndex(repoName)) if err != nil { return u, fmt.Errorf("no cached repo found. (try 'helm repo update'). %s", err) } @@ -182,7 +182,7 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, er return url.Parse(cv.URLs[0]) } -func findRepoEntry(name string, repos []*repo.Entry) (*repo.Entry, error) { +func findRepoEntry(name string, repos []*repo.ChartRepositoryConfig) (*repo.ChartRepositoryConfig, error) { for _, re := range repos { if re.Name == name { return re, nil diff --git a/cmd/helm/downloader/manager.go b/cmd/helm/downloader/manager.go index 5e55aadfb..a64ed3bf3 100644 --- a/cmd/helm/downloader/manager.go +++ b/cmd/helm/downloader/manager.go @@ -35,6 +35,7 @@ import ( "k8s.io/helm/pkg/chartutil" "k8s.io/helm/pkg/proto/hapi/chart" "k8s.io/helm/pkg/repo" + "k8s.io/helm/pkg/urlutil" ) // Manager handles the lifecycle of fetching, resolving, and storing dependencies. @@ -226,7 +227,7 @@ func (m *Manager) hasAllRepos(deps []*chartutil.Dependency) error { found = true } else { for _, repo := range repos { - if urlsAreEqual(repo.URL, strings.TrimSuffix(dd.Repository, "/")) { + if urlutil.URLAreEqual(repo.URL, strings.TrimSuffix(dd.Repository, "/")) { found = true } } @@ -258,7 +259,7 @@ func (m *Manager) getRepoNames(deps []*chartutil.Dependency) (map[string]string, found := false for _, repo := range repos { - if urlsAreEqual(repo.URL, dd.Repository) { + if urlutil.URLAreEqual(repo.URL, dd.Repository) { found = true reposMap[dd.Name] = repo.Name break @@ -283,53 +284,35 @@ func (m *Manager) UpdateRepositories() error { repos := rf.Repositories if len(repos) > 0 { // This prints warnings straight to out. - m.parallelRepoUpdate(repos) + if err := m.parallelRepoUpdate(repos); err != nil { + return err + } } return nil } -func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) { +func (m *Manager) parallelRepoUpdate(repos []*repo.ChartRepositoryConfig) error { out := m.Out fmt.Fprintln(out, "Hang tight while we grab the latest from your chart repositories...") var wg sync.WaitGroup - for _, re := range repos { + for _, c := range repos { + r, err := repo.NewChartRepository(c) + if err != nil { + return err + } wg.Add(1) - go func(n, u string) { - if err := repo.DownloadIndexFile(n, u, m.HelmHome.CacheIndex(n)); err != nil { - fmt.Fprintf(out, "...Unable to get an update from the %q chart repository (%s):\n\t%s\n", n, u, err) + go func(r *repo.ChartRepository) { + if err := r.DownloadIndexFile(); err != nil { + fmt.Fprintf(out, "...Unable to get an update from the %q chart repository (%s):\n\t%s\n", r.Config.Name, r.Config.URL, err) } else { - fmt.Fprintf(out, "...Successfully got an update from the %q chart repository\n", n) + fmt.Fprintf(out, "...Successfully got an update from the %q chart repository\n", r.Config.Name) } wg.Done() - }(re.Name, re.URL) + }(r) } wg.Wait() fmt.Fprintln(out, "Update Complete. ⎈Happy Helming!⎈") -} - -// urlsAreEqual normalizes two URLs and then compares for equality. -// -// TODO: This and the urlJoin functions should really be moved to a 'urlutil' package. -func urlsAreEqual(a, b string) bool { - au, err := url.Parse(a) - if err != nil { - a = filepath.Clean(a) - b = filepath.Clean(b) - // If urls are paths, return true only if they are an exact match - return a == b - } - bu, err := url.Parse(b) - if err != nil { - return false - } - - for _, u := range []*url.URL{au, bu} { - if u.Path == "" { - u.Path = "/" - } - u.Path = filepath.Clean(u.Path) - } - return au.String() == bu.String() + return nil } // findChartURL searches the cache of repo data for a chart that has the name and the repoURL specified. @@ -342,7 +325,7 @@ func urlsAreEqual(a, b string) bool { // If it finds a URL that is "relative", it will prepend the repoURL. func findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (string, error) { for _, cr := range repos { - if urlsAreEqual(repoURL, cr.URL) { + if urlutil.URLAreEqual(repoURL, cr.Config.URL) { entry, err := findEntryByName(name, cr) if err != nil { return "", err @@ -434,13 +417,14 @@ func (m *Manager) loadChartRepositories() (map[string]*repo.ChartRepository, err for _, re := range rf.Repositories { lname := re.Name cacheindex := m.HelmHome.CacheIndex(lname) - index, err := repo.LoadIndexFile(cacheindex) + index, err := repo.NewChartRepositoryIndexFromFile(cacheindex) if err != nil { return indices, err } + // TODO: use constructor cr := &repo.ChartRepository{ - URL: re.URL, + Config: re, IndexFile: index, } indices[lname] = cr diff --git a/cmd/helm/downloader/manager_test.go b/cmd/helm/downloader/manager_test.go index 7cfcca4ed..7224ccc41 100644 --- a/cmd/helm/downloader/manager_test.go +++ b/cmd/helm/downloader/manager_test.go @@ -135,27 +135,3 @@ func TestGetRepoNames(t *testing.T) { } } } - -func TestUrlsAreEqual(t *testing.T) { - for _, tt := range []struct { - a, b string - match bool - }{ - {"http://example.com", "http://example.com", true}, - {"http://example.com", "http://another.example.com", false}, - {"https://example.com", "https://example.com", true}, - {"http://example.com/", "http://example.com", true}, - {"https://example.com", "http://example.com", false}, - {"http://example.com/foo", "http://example.com/foo/", true}, - {"http://example.com/foo//", "http://example.com/foo/", true}, - {"http://example.com/./foo/", "http://example.com/foo/", true}, - {"http://example.com/bar/../foo/", "http://example.com/foo/", true}, - {"/foo", "/foo", true}, - {"/foo", "/foo/", true}, - {"/foo/.", "/foo/", true}, - } { - if tt.match != urlsAreEqual(tt.a, tt.b) { - t.Errorf("Expected %q==%q to be %t", tt.a, tt.b, tt.match) - } - } -} diff --git a/cmd/helm/fetch.go b/cmd/helm/fetch.go index 9a533d12e..a29b0d2ea 100644 --- a/cmd/helm/fetch.go +++ b/cmd/helm/fetch.go @@ -92,7 +92,6 @@ func newFetchCmd(out io.Writer) *cobra.Command { } func (f *fetchCmd) run() error { - pname := f.chartRef c := downloader.ChartDownloader{ HelmHome: helmpath.Home(homePath()), Out: f.out, @@ -118,7 +117,7 @@ func (f *fetchCmd) run() error { defer os.RemoveAll(dest) } - saved, v, err := c.DownloadTo(pname, f.version, dest) + saved, v, err := c.DownloadTo(f.chartRef, f.version, dest) if err != nil { return err } diff --git a/cmd/helm/helm_test.go b/cmd/helm/helm_test.go index 682709f2d..83a086875 100644 --- a/cmd/helm/helm_test.go +++ b/cmd/helm/helm_test.go @@ -254,12 +254,12 @@ func ensureTestHome(home helmpath.Home, t *testing.T) error { repoFile := home.RepositoryFile() if fi, err := os.Stat(repoFile); err != nil { - rf := repo.NewRepoFile() - rf.Add(&repo.Entry{ + rf := repo.NewRepositoryFile() + rf.Add(&repo.ChartRepositoryConfig{ Name: "charts", URL: "http://example.com/foo", Cache: "charts-index.yaml", - }, &repo.Entry{ + }, &repo.ChartRepositoryConfig{ Name: "local", URL: "http://localhost.com:7743/foo", Cache: "local-index.yaml", @@ -279,7 +279,7 @@ func ensureTestHome(home helmpath.Home, t *testing.T) error { localRepoIndexFile := home.LocalRepository(localRepoIndexFilePath) if fi, err := os.Stat(localRepoIndexFile); err != nil { - i := repo.NewIndexFile() + i := repo.NewChartRepositoryIndex() if err := i.WriteFile(localRepoIndexFile, 0644); err != nil { return err } diff --git a/cmd/helm/init.go b/cmd/helm/init.go index 11f140067..b46c32550 100644 --- a/cmd/helm/init.go +++ b/cmd/helm/init.go @@ -102,7 +102,6 @@ func newInitCmd(out io.Writer) *cobra.Command { // runInit initializes local config and installs tiller to Kubernetes Cluster func (i *initCmd) run() error { - if flagDebug { m, err := installer.DeploymentManifest(i.namespace, i.image, i.canary) if err != nil { @@ -110,13 +109,21 @@ func (i *initCmd) run() error { } fmt.Fprintln(i.out, m) } + if i.dryRun { return nil } - if err := ensureHome(i.home, i.out); err != nil { + if err := ensureDirectories(i.home, i.out); err != nil { return err } + if err := ensureDefaultRepos(i.home, i.out); err != nil { + return err + } + if err := ensureRepoFileFormat(i.home.RepositoryFile(), i.out); err != nil { + return err + } + fmt.Fprintf(i.out, "$HELM_HOME has been configured at %s.\n", helmHome) if !i.clientOnly { if i.kubeClient == nil { @@ -137,15 +144,23 @@ func (i *initCmd) run() error { } else { fmt.Fprintln(i.out, "Not installing tiller due to 'client-only' flag having been set") } + fmt.Fprintln(i.out, "Happy Helming!") return nil } -// ensureHome checks to see if $HELM_HOME exists +// ensureDirectories checks to see if $HELM_HOME exists // // If $HELM_HOME does not exist, this function will create it. -func ensureHome(home helmpath.Home, out io.Writer) error { - configDirectories := []string{home.String(), home.Repository(), home.Cache(), home.LocalRepository(), home.Plugins(), home.Starters()} +func ensureDirectories(home helmpath.Home, out io.Writer) error { + configDirectories := []string{ + home.String(), + home.Repository(), + home.Cache(), + home.LocalRepository(), + home.Plugins(), + home.Starters(), + } for _, p := range configDirectories { if fi, err := os.Stat(p); err != nil { fmt.Fprintf(out, "Creating %s \n", p) @@ -157,50 +172,77 @@ func ensureHome(home helmpath.Home, out io.Writer) error { } } + return nil +} + +func ensureDefaultRepos(home helmpath.Home, out io.Writer) error { repoFile := home.RepositoryFile() if fi, err := os.Stat(repoFile); err != nil { fmt.Fprintf(out, "Creating %s \n", repoFile) - r := repo.NewRepoFile() - r.Add(&repo.Entry{ - Name: stableRepository, - URL: stableRepositoryURL, - Cache: "stable-index.yaml", - }, &repo.Entry{ - Name: localRepository, - URL: localRepositoryURL, - Cache: "local-index.yaml", - }) - if err := r.WriteFile(repoFile, 0644); err != nil { + f := repo.NewRepositoryFile() + sr, err := initStableRepo(home.CacheIndex(stableRepository)) + if err != nil { return err } - cif := home.CacheIndex(stableRepository) - if err := repo.DownloadIndexFile(stableRepository, stableRepositoryURL, cif); err != nil { - fmt.Fprintf(out, "WARNING: Failed to download %s: %s (run 'helm repo update')\n", stableRepository, err) + lr, err := initLocalRepo(home.LocalRepository(localRepoIndexFilePath), home.CacheIndex("local")) + if err != nil { + return err } + f.Add(sr) + f.Add(lr) + f.WriteFile(repoFile, 0644) } else if fi.IsDir() { return fmt.Errorf("%s must be a file, not a directory", repoFile) } - if r, err := repo.LoadRepositoriesFile(repoFile); err == repo.ErrRepoOutOfDate { - fmt.Fprintln(out, "Updating repository file format...") - if err := r.WriteFile(repoFile, 0644); err != nil { - return err - } + return nil +} + +func initStableRepo(cacheFile string) (*repo.ChartRepositoryConfig, error) { + c := repo.ChartRepositoryConfig{ + Name: stableRepository, + URL: stableRepositoryURL, + Cache: cacheFile, + } + r, err := repo.NewChartRepository(&c) + if err != nil { + return nil, err } - localRepoIndexFile := home.LocalRepository(localRepoIndexFilePath) - if fi, err := os.Stat(localRepoIndexFile); err != nil { - fmt.Fprintf(out, "Creating %s \n", localRepoIndexFile) - i := repo.NewIndexFile() - if err := i.WriteFile(localRepoIndexFile, 0644); err != nil { - return err + if err := r.DownloadIndexFile(); err != nil { + return nil, fmt.Errorf("Looks like %q is not a valid chart repository or cannot be reached: %s", stableRepositoryURL, err.Error()) + } + + return &c, nil +} + +func initLocalRepo(indexFile, cacheFile string) (*repo.ChartRepositoryConfig, error) { + if fi, err := os.Stat(indexFile); err != nil { + i := repo.NewChartRepositoryIndex() + if err := i.WriteFile(indexFile, 0644); err != nil { + return nil, err } //TODO: take this out and replace with helm update functionality - os.Symlink(localRepoIndexFile, home.CacheIndex("local")) + os.Symlink(indexFile, cacheFile) } else if fi.IsDir() { - return fmt.Errorf("%s must be a file, not a directory", localRepoIndexFile) + return nil, fmt.Errorf("%s must be a file, not a directory", indexFile) + } + + return &repo.ChartRepositoryConfig{ + Name: localRepository, + URL: localRepositoryURL, + Cache: cacheFile, + }, nil +} + +func ensureRepoFileFormat(file string, out io.Writer) error { + r, err := repo.LoadRepositoriesFile(file) + if err == repo.ErrRepoOutOfDate { + fmt.Fprintln(out, "Updating repository file format...") + if err := r.WriteFile(file, 0644); err != nil { + return err + } } - fmt.Fprintf(out, "$HELM_HOME has been configured at %s.\n", helmHome) return nil } diff --git a/cmd/helm/init_test.go b/cmd/helm/init_test.go index 7ad617570..d30b985aa 100644 --- a/cmd/helm/init_test.go +++ b/cmd/helm/init_test.go @@ -43,20 +43,22 @@ func TestInitCmd(t *testing.T) { defer os.Remove(home) var buf bytes.Buffer - fc := fake.NewSimpleClientset() + + fake := testclient.Fake{} cmd := &initCmd{ out: &buf, home: helmpath.Home(home), - kubeClient: fc.Extensions(), - namespace: api.NamespaceDefault, + kubeClient: fake.Extensions(), } if err := cmd.run(); err != nil { t.Errorf("expected error: %v", err) } - action := fc.Actions()[0] - if !action.Matches("create", "deployments") { - t.Errorf("unexpected action: %v, expected create deployment", action) + + actions := fake.Actions() + if action, ok := actions[0].(testclient.CreateAction); !ok || action.GetResource() != "deployments" { + t.Errorf("unexpected action: %v, expected create deployment", actions[0]) } + expected := "Tiller (the helm server side component) has been installed into your Kubernetes Cluster." if !strings.Contains(buf.String(), expected) { t.Errorf("expected %q, got %q", expected, buf.String()) @@ -169,7 +171,13 @@ func TestEnsureHome(t *testing.T) { b := bytes.NewBuffer(nil) hh := helmpath.Home(home) helmHome = home - if err := ensureHome(hh, b); err != nil { + if err := ensureDirectories(hh, b); err != nil { + t.Error(err) + } + if err := ensureDefaultRepos(hh, b); err != nil { + t.Error(err) + } + if err := ensureRepoFileFormat(hh.RepositoryFile(), b); err != nil { t.Error(err) } diff --git a/cmd/helm/repo_add.go b/cmd/helm/repo_add.go index d36b48488..0abaa9740 100644 --- a/cmd/helm/repo_add.go +++ b/cmd/helm/repo_add.go @@ -19,8 +19,6 @@ package main import ( "fmt" "io" - "path/filepath" - "strings" "github.com/spf13/cobra" @@ -32,8 +30,13 @@ type repoAddCmd struct { name string url string home helmpath.Home - out io.Writer noupdate bool + + certFile string + keyFile string + caFile string + + out io.Writer } func newRepoAddCmd(out io.Writer) *cobra.Command { @@ -56,73 +59,54 @@ func newRepoAddCmd(out io.Writer) *cobra.Command { return add.run() }, } + f := cmd.Flags() f.BoolVar(&add.noupdate, "no-update", false, "raise error if repo is already registered") + f.StringVar(&add.certFile, "cert-file", "", "identify HTTPS client using this SSL certificate file") + f.StringVar(&add.keyFile, "key-file", "", "identify HTTPS client using this SSL key file") + f.StringVar(&add.caFile, "ca-file", "", "verify certificates of HTTPS-enabled servers using this CA bundle") + return cmd } func (a *repoAddCmd) run() error { - var err error - if a.noupdate { - err = addRepository(a.name, a.url, a.home) - } else { - err = updateRepository(a.name, a.url, a.home) - } - if err != nil { + if err := addRepository(a.name, a.url, a.home, a.certFile, a.keyFile, a.caFile, a.noupdate); err != nil { return err } fmt.Fprintf(a.out, "%q has been added to your repositories\n", a.name) return nil } -func addRepository(name, url string, home helmpath.Home) error { - cif := home.CacheIndex(name) - if err := repo.DownloadIndexFile(name, url, cif); err != nil { - return fmt.Errorf("Looks like %q is not a valid chart repository or cannot be reached: %s", url, err.Error()) - } - - return insertRepoLine(name, url, home) -} - -func insertRepoLine(name, url string, home helmpath.Home) error { - cif := home.CacheIndex(name) +func addRepository(name, url string, home helmpath.Home, certFile, keyFile, caFile string, noUpdate bool) error { f, err := repo.LoadRepositoriesFile(home.RepositoryFile()) if err != nil { return err } - if f.Has(name) { + if noUpdate && f.Has(name) { return fmt.Errorf("The repository name you provided (%s) already exists. Please specify a different name.", name) } - f.Add(&repo.Entry{ - Name: name, - URL: strings.TrimSuffix(url, "/"), - Cache: filepath.Base(cif), - }) - return f.WriteFile(home.RepositoryFile(), 0644) -} -func updateRepository(name, url string, home helmpath.Home) error { cif := home.CacheIndex(name) - if err := repo.DownloadIndexFile(name, url, cif); err != nil { - return err + c := repo.ChartRepositoryConfig{ + Name: name, + Cache: cif, + URL: url, + CertFile: certFile, + KeyFile: keyFile, + CAFile: caFile, } - return updateRepoLine(name, url, home) -} - -func updateRepoLine(name, url string, home helmpath.Home) error { - cif := home.CacheIndex(name) - f, err := repo.LoadRepositoriesFile(home.RepositoryFile()) + r, err := repo.NewChartRepository(&c) if err != nil { return err } - f.Update(&repo.Entry{ - Name: name, - URL: url, - Cache: filepath.Base(cif), - }) + if err := r.DownloadIndexFile(); err != nil { + return fmt.Errorf("Looks like %q is not a valid chart repository or cannot be reached: %s", url, err.Error()) + } + + f.Update(&c) - return f.WriteFile(home.RepositoryFile(), 0666) + return f.WriteFile(home.RepositoryFile(), 0644) } diff --git a/cmd/helm/repo_add_test.go b/cmd/helm/repo_add_test.go index c9c255433..7c5342177 100644 --- a/cmd/helm/repo_add_test.go +++ b/cmd/helm/repo_add_test.go @@ -80,7 +80,7 @@ func TestRepoAdd(t *testing.T) { t.Fatal(err) } - if err := addRepository(testName, ts.URL(), hh); err != nil { + if err := addRepository(testName, ts.URL(), hh, "", "", "", true); err != nil { t.Error(err) } @@ -93,11 +93,11 @@ func TestRepoAdd(t *testing.T) { t.Errorf("%s was not successfully inserted into %s", testName, hh.RepositoryFile()) } - if err := updateRepository(testName, ts.URL(), hh); err != nil { + if err := addRepository(testName, ts.URL(), hh, "", "", "", false); err != nil { t.Errorf("Repository was not updated: %s", err) } - if err := addRepository(testName, ts.URL(), hh); err == nil { + if err := addRepository(testName, ts.URL(), hh, "", "", "", false); err != nil { t.Errorf("Duplicate repository name was added") } } diff --git a/cmd/helm/repo_index.go b/cmd/helm/repo_index.go index ad5808946..925dffc17 100644 --- a/cmd/helm/repo_index.go +++ b/cmd/helm/repo_index.go @@ -83,12 +83,12 @@ func (i *repoIndexCmd) run() error { func index(dir, url, mergeTo string) error { out := filepath.Join(dir, "index.yaml") - i, err := repo.IndexDirectory(dir, url) + i, err := repo.NewChartRepositoryIndexFromDirectory(dir, url) if err != nil { return err } if mergeTo != "" { - i2, err := repo.LoadIndexFile(mergeTo) + i2, err := repo.NewChartRepositoryIndexFromFile(mergeTo) if err != nil { return fmt.Errorf("Merge failed: %s", err) } diff --git a/cmd/helm/repo_index_test.go b/cmd/helm/repo_index_test.go index bd1010dd7..b3e37e4b1 100644 --- a/cmd/helm/repo_index_test.go +++ b/cmd/helm/repo_index_test.go @@ -53,7 +53,7 @@ func TestRepoIndexCmd(t *testing.T) { destIndex := filepath.Join(dir, "index.yaml") - index, err := repo.LoadIndexFile(destIndex) + index, err := repo.NewChartRepositoryIndexFromFile(destIndex) if err != nil { t.Fatal(err) } @@ -94,7 +94,7 @@ func TestRepoIndexCmd(t *testing.T) { t.Error(err) } - index, err = repo.LoadIndexFile(destIndex) + index, err = repo.NewChartRepositoryIndexFromFile(destIndex) if err != nil { t.Fatal(err) } diff --git a/cmd/helm/repo_remove_test.go b/cmd/helm/repo_remove_test.go index 9f2421b29..12f534972 100644 --- a/cmd/helm/repo_remove_test.go +++ b/cmd/helm/repo_remove_test.go @@ -24,24 +24,33 @@ import ( "k8s.io/helm/cmd/helm/helmpath" "k8s.io/helm/pkg/repo" + "k8s.io/helm/pkg/repo/repotest" ) func TestRepoRemove(t *testing.T) { - testURL := "https://test-url.com" - - b := bytes.NewBuffer(nil) - - home, err := tempHelmHome(t) + ts, thome, err := repotest.NewTempServer("testdata/testserver/*.*") if err != nil { t.Fatal(err) } - defer os.Remove(home) - hh := helmpath.Home(home) + + oldhome := homePath() + helmHome = thome + hh := helmpath.Home(thome) + defer func() { + ts.Stop() + helmHome = oldhome + os.Remove(thome) + }() + if err := ensureTestHome(hh, t); err != nil { + t.Fatal(err) + } + + b := bytes.NewBuffer(nil) if err := removeRepoLine(b, testName, hh); err == nil { t.Errorf("Expected error removing %s, but did not get one.", testName) } - if err := insertRepoLine(testName, testURL, hh); err != nil { + if err := addRepository(testName, ts.URL(), hh, "", "", "", true); err != nil { t.Error(err) } diff --git a/cmd/helm/repo_update.go b/cmd/helm/repo_update.go index 7e04b5e27..8dfab9a5d 100644 --- a/cmd/helm/repo_update.go +++ b/cmd/helm/repo_update.go @@ -36,10 +36,14 @@ Information is cached locally, where it is used by commands like 'helm search'. future releases. ` +var ( + errNoRepositories = errors.New("no repositories found. You must add one before updating") +) + type repoUpdateCmd struct { - update func([]*repo.Entry, bool, io.Writer, helmpath.Home) - out io.Writer + update func([]*repo.ChartRepository, io.Writer) home helmpath.Home + out io.Writer } func newRepoUpdateCmd(out io.Writer) *cobra.Command { @@ -67,31 +71,39 @@ func (u *repoUpdateCmd) run() error { } if len(f.Repositories) == 0 { - return errors.New("no repositories found. You must add one before updating") + return errNoRepositories + } + var repos []*repo.ChartRepository + for _, cfg := range f.Repositories { + r, err := repo.NewChartRepository(cfg) + if err != nil { + return err + } + repos = append(repos, r) } - u.update(f.Repositories, flagDebug, u.out, u.home) + u.update(repos, u.out) return nil } -func updateCharts(repos []*repo.Entry, verbose bool, out io.Writer, home helmpath.Home) { +func updateCharts(repos []*repo.ChartRepository, out io.Writer) { fmt.Fprintln(out, "Hang tight while we grab the latest from your chart repositories...") var wg sync.WaitGroup for _, re := range repos { wg.Add(1) - go func(n, u string) { + go func(re *repo.ChartRepository) { defer wg.Done() - if n == localRepository { - // We skip local because the indices are symlinked. + if re.Config.Name == localRepository { + fmt.Fprintf(out, "...Skip %s chart repository", re.Config.Name) return } - err := repo.DownloadIndexFile(n, u, home.CacheIndex(n)) + err := re.DownloadIndexFile() if err != nil { - fmt.Fprintf(out, "...Unable to get an update from the %q chart repository (%s):\n\t%s\n", n, u, err) + fmt.Fprintf(out, "...Unable to get an update from the %q chart repository (%s):\n\t%s\n", re.Config.Name, re.Config.URL, err) } else { - fmt.Fprintf(out, "...Successfully got an update from the %q chart repository\n", n) + fmt.Fprintf(out, "...Successfully got an update from the %q chart repository\n", re.Config.Name) } - }(re.Name, re.URL) + }(re) } wg.Wait() fmt.Fprintln(out, "Update Complete. ⎈ Happy Helming!⎈ ") diff --git a/cmd/helm/repo_update_test.go b/cmd/helm/repo_update_test.go index 5906bd62d..f7be052b6 100644 --- a/cmd/helm/repo_update_test.go +++ b/cmd/helm/repo_update_test.go @@ -43,15 +43,15 @@ func TestUpdateCmd(t *testing.T) { out := bytes.NewBuffer(nil) // Instead of using the HTTP updater, we provide our own for this test. // The TestUpdateCharts test verifies the HTTP behavior independently. - updater := func(repos []*repo.Entry, verbose bool, out io.Writer, home helmpath.Home) { + updater := func(repos []*repo.ChartRepository, out io.Writer) { for _, re := range repos { - fmt.Fprintln(out, re.Name) + fmt.Fprintln(out, re.Config.Name) } } uc := &repoUpdateCmd{ - out: out, update: updater, home: helmpath.Home(thome), + out: out, } if err := uc.run(); err != nil { t.Fatal(err) @@ -63,33 +63,40 @@ func TestUpdateCmd(t *testing.T) { } func TestUpdateCharts(t *testing.T) { - srv, thome, err := repotest.NewTempServer("testdata/testserver/*.*") + ts, thome, err := repotest.NewTempServer("testdata/testserver/*.*") if err != nil { t.Fatal(err) } oldhome := homePath() helmHome = thome + hh := helmpath.Home(thome) defer func() { - srv.Stop() + ts.Stop() helmHome = oldhome os.Remove(thome) }() - if err := ensureTestHome(helmpath.Home(thome), t); err != nil { + if err := ensureTestHome(hh, t); err != nil { t.Fatal(err) } - buf := bytes.NewBuffer(nil) - repos := []*repo.Entry{ - {Name: "charts", URL: srv.URL()}, + r, err := repo.NewChartRepository(&repo.ChartRepositoryConfig{ + Name: "charts", + URL: ts.URL(), + Cache: hh.CacheIndex("charts"), + }) + if err != nil { + t.Error(err) } - updateCharts(repos, false, buf, helmpath.Home(thome)) - got := buf.String() + b := bytes.NewBuffer(nil) + updateCharts([]*repo.ChartRepository{r}, b) + + got := b.String() if strings.Contains(got, "Unable to get an update") { t.Errorf("Failed to get a repo: %q", got) } if !strings.Contains(got, "Update Complete.") { - t.Errorf("Update was not successful") + t.Error("Update was not successful") } } diff --git a/cmd/helm/resolver/resolver.go b/cmd/helm/resolver/resolver.go index 480cea674..dfa1e74a7 100644 --- a/cmd/helm/resolver/resolver.go +++ b/cmd/helm/resolver/resolver.go @@ -60,7 +60,7 @@ func (r *Resolver) Resolve(reqs *chartutil.Requirements, repoNames map[string]st return nil, fmt.Errorf("dependency %q has an invalid version/constraint format: %s", d.Name, err) } - repoIndex, err := repo.LoadIndexFile(r.helmhome.CacheIndex(repoNames[d.Name])) + repoIndex, err := repo.NewChartRepositoryIndexFromFile(r.helmhome.CacheIndex(repoNames[d.Name])) if err != nil { return nil, fmt.Errorf("no cached repo found. (try 'helm repo update'). %s", err) } diff --git a/cmd/helm/search.go b/cmd/helm/search.go index a1d57adb0..62d13e3b3 100644 --- a/cmd/helm/search.go +++ b/cmd/helm/search.go @@ -119,7 +119,7 @@ func (s *searchCmd) buildIndex() (*search.Index, error) { for _, re := range rf.Repositories { n := re.Name f := s.helmhome.CacheIndex(n) - ind, err := repo.LoadIndexFile(f) + ind, err := repo.NewChartRepositoryIndexFromFile(f) if err != nil { fmt.Fprintf(s.out, "WARNING: Repo %q is corrupt or missing. Try 'helm repo update'.", n) continue diff --git a/cmd/helm/search/search.go b/cmd/helm/search/search.go index 4d394e0f5..5675332df 100644 --- a/cmd/helm/search/search.go +++ b/cmd/helm/search/search.go @@ -61,7 +61,7 @@ func NewIndex() *Index { const verSep = "$$" // AddRepo adds a repository index to the search index. -func (i *Index) AddRepo(rname string, ind *repo.IndexFile, all bool) { +func (i *Index) AddRepo(rname string, ind *repo.ChartRepositoryIndex, all bool) { for name, ref := range ind.Entries { if len(ref) == 0 { // Skip chart names that have zero releases. diff --git a/cmd/helm/search/search_test.go b/cmd/helm/search/search_test.go index 7f5d29409..1a607fdc7 100644 --- a/cmd/helm/search/search_test.go +++ b/cmd/helm/search/search_test.go @@ -95,8 +95,8 @@ var indexfileEntries = map[string]repo.ChartVersions{ func loadTestIndex(t *testing.T, all bool) *Index { i := NewIndex() - i.AddRepo("testing", &repo.IndexFile{Entries: indexfileEntries}, all) - i.AddRepo("ztesting", &repo.IndexFile{Entries: map[string]repo.ChartVersions{ + i.AddRepo("testing", &repo.ChartRepositoryIndex{Entries: indexfileEntries}, all) + i.AddRepo("ztesting", &repo.ChartRepositoryIndex{Entries: map[string]repo.ChartVersions{ "pinta": { { URLs: []string{"http://example.com/charts/pinta-2.0.0.tgz"}, diff --git a/pkg/repo/chartrepo.go b/pkg/repo/chartrepo.go new file mode 100644 index 000000000..c83592678 --- /dev/null +++ b/pkg/repo/chartrepo.go @@ -0,0 +1,168 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repo // import "k8s.io/helm/pkg/repo" + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/ghodss/yaml" + + "k8s.io/helm/pkg/chartutil" + "k8s.io/helm/pkg/provenance" + "k8s.io/helm/pkg/tlsutil" +) + +// ChartRepositoryConfig represents a collection of parameters for chart repository +type ChartRepositoryConfig struct { + Name string `json:"name"` + Cache string `json:"cache"` + URL string `json:"url"` + CertFile string `json:"certFile"` + KeyFile string `json:"keyFile"` + CAFile string `json:"caFile"` +} + +// ChartRepository represents a chart repository +type ChartRepository struct { + Config *ChartRepositoryConfig + ChartPaths []string + IndexFile *ChartRepositoryIndex + Client *http.Client +} + +// NewChartRepository constructs ChartRepository +func NewChartRepository(cfg *ChartRepositoryConfig) (*ChartRepository, error) { + var client *http.Client + if cfg.CertFile != "" && cfg.KeyFile != "" && cfg.CAFile != "" { + tlsConf, err := tlsutil.NewClientTLS(cfg.CertFile, cfg.KeyFile, cfg.CAFile) + if err != nil { + return nil, fmt.Errorf("can't create TLS config for client: %s", err.Error()) + } + tlsConf.BuildNameToCertificate() + client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConf, + }, + } + } else { + client = http.DefaultClient + } + + return &ChartRepository{ + Config: cfg, + IndexFile: NewChartRepositoryIndex(), + Client: client, + }, nil +} + +// Load loads a directory of charts as if it were a repository. +// +// It requires the presence of an index.yaml file in the directory. +func (r *ChartRepository) Load() error { + dirInfo, err := os.Stat(r.Config.Name) + if err != nil { + return err + } + if !dirInfo.IsDir() { + return fmt.Errorf("%q is not a directory", r.Config.Name) + } + + // FIXME: Why are we recursively walking directories? + // FIXME: Why are we not reading the repositories.yaml to figure out + // what repos to use? + filepath.Walk(r.Config.Name, func(path string, f os.FileInfo, err error) error { + if !f.IsDir() { + if strings.Contains(f.Name(), "-index.yaml") { + i, err := NewChartRepositoryIndexFromFile(path) + if err != nil { + return nil + } + r.IndexFile = i + } else if strings.HasSuffix(f.Name(), ".tgz") { + r.ChartPaths = append(r.ChartPaths, path) + } + } + return nil + }) + return nil +} + +// DownloadIndexFile fetches the index from a repository. +func (r *ChartRepository) DownloadIndexFile() error { + var indexURL string + + indexURL = strings.TrimSuffix(r.Config.URL, "/") + "/index.yaml" + resp, err := r.Client.Get(indexURL) + if err != nil { + return err + } + defer resp.Body.Close() + + index, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + if _, err := loadIndex(index); err != nil { + return err + } + + return ioutil.WriteFile(r.Config.Cache, index, 0644) +} + +// Index generates an index for the chart repository and writes an index.yaml file. +func (r *ChartRepository) Index() error { + err := r.generateIndex() + if err != nil { + return err + } + return r.saveIndexFile() +} + +func (r *ChartRepository) saveIndexFile() error { + index, err := yaml.Marshal(r.IndexFile) + if err != nil { + return err + } + return ioutil.WriteFile(filepath.Join(r.Config.Name, indexPath), index, 0644) +} + +func (r *ChartRepository) generateIndex() error { + for _, path := range r.ChartPaths { + ch, err := chartutil.Load(path) + if err != nil { + return err + } + + digest, err := provenance.DigestFile(path) + if err != nil { + return err + } + + if !r.IndexFile.Has(ch.Metadata.Name, ch.Metadata.Version) { + r.IndexFile.Add(ch.Metadata, path, r.Config.URL, digest) + } + // TODO: If a chart exists, but has a different Digest, should we error? + } + r.IndexFile.SortEntries() + return nil +} diff --git a/pkg/repo/chartrepo_test.go b/pkg/repo/chartrepo_test.go new file mode 100644 index 000000000..b9a83a4c2 --- /dev/null +++ b/pkg/repo/chartrepo_test.go @@ -0,0 +1,185 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repo + +import ( + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "k8s.io/helm/pkg/proto/hapi/chart" +) + +const ( + testRepository = "testdata/repository" + testURL = "http://example-charts.com" +) + +func TestLoadChartRepository(t *testing.T) { + r, err := NewChartRepository(&ChartRepositoryConfig{ + Name: testRepository, + URL: testURL, + }) + if err != nil { + t.Errorf("Problem creating chart repository from %s: %v", testRepository, err) + } + + if err := r.Load(); err != nil { + t.Errorf("Problem loading chart repository from %s: %v", testRepository, err) + } + + paths := []string{ + filepath.Join(testRepository, "frobnitz-1.2.3.tgz"), + filepath.Join(testRepository, "sprocket-1.1.0.tgz"), + filepath.Join(testRepository, "sprocket-1.2.0.tgz"), + } + + if r.Config.Name != testRepository { + t.Errorf("Expected %s as Name but got %s", testRepository, r.Config.Name) + } + + if !reflect.DeepEqual(r.ChartPaths, paths) { + t.Errorf("Expected %#v but got %#v\n", paths, r.ChartPaths) + } + + if r.Config.URL != testURL { + t.Errorf("Expected url for chart repository to be %s but got %s", testURL, r.Config.URL) + } +} + +func TestIndex(t *testing.T) { + r, err := NewChartRepository(&ChartRepositoryConfig{ + Name: testRepository, + URL: testURL, + }) + if err != nil { + t.Errorf("Problem creating chart repository from %s: %v", testRepository, err) + } + + if err := r.Load(); err != nil { + t.Errorf("Problem loading chart repository from %s: %v", testRepository, err) + } + + err = r.Index() + if err != nil { + t.Errorf("Error performing index: %v\n", err) + } + + tempIndexPath := filepath.Join(testRepository, indexPath) + actual, err := NewChartRepositoryIndexFromFile(tempIndexPath) + defer os.Remove(tempIndexPath) // clean up + if err != nil { + t.Errorf("Error loading index file %v", err) + } + verifyIndex(t, actual) + + // Re-index and test again. + err = r.Index() + if err != nil { + t.Errorf("Error performing re-index: %s\n", err) + } + second, err := NewChartRepositoryIndexFromFile(tempIndexPath) + if err != nil { + t.Errorf("Error re-loading index file %v", err) + } + verifyIndex(t, second) +} + +func verifyIndex(t *testing.T, actual *ChartRepositoryIndex) { + var empty time.Time + if actual.Generated == empty { + t.Errorf("Generated should be greater than 0: %s", actual.Generated) + } + + if actual.APIVersion != APIVersionV1 { + t.Error("Expected v1 API") + } + + entries := actual.Entries + if numEntries := len(entries); numEntries != 2 { + t.Errorf("Expected 2 charts to be listed in index file but got %v", numEntries) + } + + expects := map[string]ChartVersions{ + "frobnitz": { + { + Metadata: &chart.Metadata{ + Name: "frobnitz", + Version: "1.2.3", + }, + }, + }, + "sprocket": { + { + Metadata: &chart.Metadata{ + Name: "sprocket", + Version: "1.2.0", + }, + }, + { + Metadata: &chart.Metadata{ + Name: "sprocket", + Version: "1.1.0", + }, + }, + }, + } + + for name, versions := range expects { + got, ok := entries[name] + if !ok { + t.Errorf("Could not find %q entry", name) + continue + } + if len(versions) != len(got) { + t.Errorf("Expected %d versions, got %d", len(versions), len(got)) + continue + } + for i, e := range versions { + g := got[i] + if e.Name != g.Name { + t.Errorf("Expected %q, got %q", e.Name, g.Name) + } + if e.Version != g.Version { + t.Errorf("Expected %q, got %q", e.Version, g.Version) + } + if len(g.Keywords) != 3 { + t.Error("Expected 3 keyrwords.") + } + if len(g.Maintainers) != 2 { + t.Error("Expected 2 maintainers.") + } + if g.Created == empty { + t.Error("Expected created to be non-empty") + } + if g.Description == "" { + t.Error("Expected description to be non-empty") + } + if g.Home == "" { + t.Error("Expected home to be non-empty") + } + if g.Digest == "" { + t.Error("Expected digest to be non-empty") + } + if len(g.URLs) != 1 { + t.Error("Expected exactly 1 URL") + } + } + } +} diff --git a/pkg/repo/index.go b/pkg/repo/index.go index f8c27f004..1676b118e 100644 --- a/pkg/repo/index.go +++ b/pkg/repo/index.go @@ -21,10 +21,7 @@ import ( "errors" "fmt" "io/ioutil" - "net/http" - "net/url" "os" - "path" "path/filepath" "sort" "strings" @@ -36,6 +33,7 @@ import ( "k8s.io/helm/pkg/chartutil" "k8s.io/helm/pkg/proto/hapi/chart" "k8s.io/helm/pkg/provenance" + "k8s.io/helm/pkg/urlutil" ) var indexPath = "index.yaml" @@ -76,17 +74,35 @@ func (c ChartVersions) Less(a, b int) bool { return i.LessThan(j) } -// IndexFile represents the index file in a chart repository -type IndexFile struct { +// ChartRepositoryIndex represents the index file in a chart repository +type ChartRepositoryIndex struct { APIVersion string `json:"apiVersion"` Generated time.Time `json:"generated"` Entries map[string]ChartVersions `json:"entries"` PublicKeys []string `json:"publicKeys,omitempty"` } -// NewIndexFile initializes an index. -func NewIndexFile() *IndexFile { - return &IndexFile{ +// ChartVersion represents a chart entry in the ChartRepositoryIndex +type ChartVersion struct { + *chart.Metadata + URLs []string `json:"urls"` + Created time.Time `json:"created,omitempty"` + Removed bool `json:"removed,omitempty"` + Digest string `json:"digest,omitempty"` +} + +// unversionedEntry represents a deprecated pre-Alpha.5 format. +// +// This will be removed prior to v2.0.0 +type unversionedEntry struct { + Checksum string `json:"checksum"` + URL string `json:"url"` + Chartfile *chart.Metadata `json:"chartfile"` +} + +// NewChartRepositoryIndex initializes an index. +func NewChartRepositoryIndex() *ChartRepositoryIndex { + return &ChartRepositoryIndex{ APIVersion: APIVersionV1, Generated: time.Now(), Entries: map[string]ChartVersions{}, @@ -94,14 +110,103 @@ func NewIndexFile() *IndexFile { } } +// NewChartRepositoryIndexFromFile takes a file at the given path and returns an ChartRepositoryIndex object +func NewChartRepositoryIndexFromFile(path string) (*ChartRepositoryIndex, error) { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + return loadIndex(b) +} + +// NewChartRepositoryIndexFromDirectory reads a (flat) directory and generates an index. +// +// It indexes only charts that have been packaged (*.tgz). +// +// The index returned will be in an unsorted state +func NewChartRepositoryIndexFromDirectory(dir, baseURL string) (*ChartRepositoryIndex, error) { + archives, err := filepath.Glob(filepath.Join(dir, "*.tgz")) + if err != nil { + return nil, err + } + index := NewChartRepositoryIndex() + for _, arch := range archives { + fname := filepath.Base(arch) + c, err := chartutil.Load(arch) + if err != nil { + // Assume this is not a chart. + continue + } + hash, err := provenance.DigestFile(arch) + if err != nil { + return index, err + } + index.Add(c.Metadata, fname, baseURL, hash) + } + return index, nil +} + +// loadIndex loads an index file and does minimal validity checking. +// +// This will fail if API Version is not set (ErrNoAPIVersion) or if the unmarshal fails. +func loadIndex(data []byte) (*ChartRepositoryIndex, error) { + i := &ChartRepositoryIndex{} + if err := yaml.Unmarshal(data, i); err != nil { + return i, err + } + if i.APIVersion == "" { + // When we leave Beta, we should remove legacy support and just + // return this error: + //return i, ErrNoAPIVersion + return loadUnversionedIndex(data) + } + return i, nil +} + +// loadUnversionedIndex loads a pre-Alpha.5 index.yaml file. +// +// This format is deprecated. This function will be removed prior to v2.0.0. +func loadUnversionedIndex(data []byte) (*ChartRepositoryIndex, error) { + fmt.Fprintln(os.Stderr, "WARNING: Deprecated index file format. Try 'helm repo update'") + i := map[string]unversionedEntry{} + + // This gets around an error in the YAML parser. Instead of parsing as YAML, + // we convert to JSON, and then decode again. + var err error + data, err = yaml.YAMLToJSON(data) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &i); err != nil { + return nil, err + } + + if len(i) == 0 { + return nil, ErrNoAPIVersion + } + ni := NewChartRepositoryIndex() + for n, item := range i { + if item.Chartfile == nil || item.Chartfile.Name == "" { + parts := strings.Split(n, "-") + ver := "" + if len(parts) > 1 { + ver = strings.TrimSuffix(parts[1], ".tgz") + } + item.Chartfile = &chart.Metadata{Name: parts[0], Version: ver} + } + ni.Add(item.Chartfile, item.URL, "", item.Checksum) + } + return ni, nil +} + // Add adds a file to the index // This can leave the index in an unsorted state -func (i IndexFile) Add(md *chart.Metadata, filename, baseURL, digest string) { +func (i ChartRepositoryIndex) Add(md *chart.Metadata, filename, baseURL, digest string) { u := filename if baseURL != "" { var err error _, file := filepath.Split(filename) - u, err = urlJoin(baseURL, file) + u, err = urlutil.URLJoin(baseURL, file) if err != nil { u = filepath.Join(baseURL, file) } @@ -120,7 +225,7 @@ func (i IndexFile) Add(md *chart.Metadata, filename, baseURL, digest string) { } // Has returns true if the index has an entry for a chart with the given name and exact version. -func (i IndexFile) Has(name, version string) bool { +func (i ChartRepositoryIndex) Has(name, version string) bool { _, err := i.Get(name, version) return err == nil } @@ -131,7 +236,7 @@ func (i IndexFile) Has(name, version string) bool { // the most recent release for every version is in the 0th slot in the // Entries.ChartVersions array. That way, tooling can predict the newest // version without needing to parse SemVers. -func (i IndexFile) SortEntries() { +func (i ChartRepositoryIndex) SortEntries() { for _, versions := range i.Entries { sort.Sort(sort.Reverse(versions)) } @@ -140,7 +245,7 @@ func (i IndexFile) SortEntries() { // Get returns the ChartVersion for the given name. // // If version is empty, this will return the chart with the highest version. -func (i IndexFile) Get(name, version string) (*ChartVersion, error) { +func (i ChartRepositoryIndex) Get(name, version string) (*ChartVersion, error) { vs, ok := i.Entries[name] if !ok { return nil, ErrNoChartName @@ -163,7 +268,7 @@ func (i IndexFile) Get(name, version string) (*ChartVersion, error) { // WriteFile writes an index file to the given destination path. // // The mode on the file is set to 'mode'. -func (i IndexFile) WriteFile(dest string, mode os.FileMode) error { +func (i ChartRepositoryIndex) WriteFile(dest string, mode os.FileMode) error { b, err := yaml.Marshal(i) if err != nil { return err @@ -179,7 +284,7 @@ func (i IndexFile) WriteFile(dest string, mode os.FileMode) error { // In all other cases, the existing record is preserved. // // This can leave the index in an unsorted state -func (i *IndexFile) Merge(f *IndexFile) { +func (i *ChartRepositoryIndex) Merge(f *ChartRepositoryIndex) { for _, cvs := range f.Entries { for _, cv := range cvs { if !i.Has(cv.Name, cv.Version) { @@ -189,153 +294,3 @@ func (i *IndexFile) Merge(f *IndexFile) { } } } - -// Need both JSON and YAML annotations until we get rid of gopkg.in/yaml.v2 - -// ChartVersion represents a chart entry in the IndexFile -type ChartVersion struct { - *chart.Metadata - URLs []string `json:"urls"` - Created time.Time `json:"created,omitempty"` - Removed bool `json:"removed,omitempty"` - Digest string `json:"digest,omitempty"` -} - -// IndexDirectory reads a (flat) directory and generates an index. -// -// It indexes only charts that have been packaged (*.tgz). -// -// The index returned will be in an unsorted state -func IndexDirectory(dir, baseURL string) (*IndexFile, error) { - archives, err := filepath.Glob(filepath.Join(dir, "*.tgz")) - if err != nil { - return nil, err - } - index := NewIndexFile() - for _, arch := range archives { - fname := filepath.Base(arch) - c, err := chartutil.Load(arch) - if err != nil { - // Assume this is not a chart. - continue - } - hash, err := provenance.DigestFile(arch) - if err != nil { - return index, err - } - index.Add(c.Metadata, fname, baseURL, hash) - } - return index, nil -} - -// DownloadIndexFile fetches the index from a repository. -func DownloadIndexFile(repoName, url, indexFilePath string) error { - var indexURL string - - indexURL = strings.TrimSuffix(url, "/") + "/index.yaml" - resp, err := http.Get(indexURL) - if err != nil { - return err - } - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - if _, err := LoadIndex(b); err != nil { - return err - } - - return ioutil.WriteFile(indexFilePath, b, 0644) -} - -// LoadIndex loads an index file and does minimal validity checking. -// -// This will fail if API Version is not set (ErrNoAPIVersion) or if the unmarshal fails. -func LoadIndex(data []byte) (*IndexFile, error) { - i := &IndexFile{} - if err := yaml.Unmarshal(data, i); err != nil { - return i, err - } - if i.APIVersion == "" { - // When we leave Beta, we should remove legacy support and just - // return this error: - //return i, ErrNoAPIVersion - return loadUnversionedIndex(data) - } - return i, nil -} - -// unversionedEntry represents a deprecated pre-Alpha.5 format. -// -// This will be removed prior to v2.0.0 -type unversionedEntry struct { - Checksum string `json:"checksum"` - URL string `json:"url"` - Chartfile *chart.Metadata `json:"chartfile"` -} - -// loadUnversionedIndex loads a pre-Alpha.5 index.yaml file. -// -// This format is deprecated. This function will be removed prior to v2.0.0. -func loadUnversionedIndex(data []byte) (*IndexFile, error) { - fmt.Fprintln(os.Stderr, "WARNING: Deprecated index file format. Try 'helm repo update'") - i := map[string]unversionedEntry{} - - // This gets around an error in the YAML parser. Instead of parsing as YAML, - // we convert to JSON, and then decode again. - var err error - data, err = yaml.YAMLToJSON(data) - if err != nil { - return nil, err - } - if err := json.Unmarshal(data, &i); err != nil { - return nil, err - } - - if len(i) == 0 { - return nil, ErrNoAPIVersion - } - ni := NewIndexFile() - for n, item := range i { - if item.Chartfile == nil || item.Chartfile.Name == "" { - parts := strings.Split(n, "-") - ver := "" - if len(parts) > 1 { - ver = strings.TrimSuffix(parts[1], ".tgz") - } - item.Chartfile = &chart.Metadata{Name: parts[0], Version: ver} - } - ni.Add(item.Chartfile, item.URL, "", item.Checksum) - } - return ni, nil -} - -// LoadIndexFile takes a file at the given path and returns an IndexFile object -func LoadIndexFile(path string) (*IndexFile, error) { - b, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - return LoadIndex(b) -} - -// urlJoin joins a base URL to one or more path components. -// -// It's like filepath.Join for URLs. If the baseURL is pathish, this will still -// perform a join. -// -// If the URL is unparsable, this returns an error. -func urlJoin(baseURL string, paths ...string) (string, error) { - u, err := url.Parse(baseURL) - if err != nil { - return "", err - } - // We want path instead of filepath because path always uses /. - all := []string{u.Path} - all = append(all, paths...) - u.Path = path.Join(all...) - return u.String(), nil -} diff --git a/pkg/repo/index_test.go b/pkg/repo/index_test.go index cbeb01bcb..7ba4da6c5 100644 --- a/pkg/repo/index_test.go +++ b/pkg/repo/index_test.go @@ -33,7 +33,7 @@ const ( ) func TestIndexFile(t *testing.T) { - i := NewIndexFile() + i := NewChartRepositoryIndex() i.Add(&chart.Metadata{Name: "clipper", Version: "0.1.0"}, "clipper-0.1.0.tgz", "http://example.com/charts", "sha256:1234567890") i.Add(&chart.Metadata{Name: "cutter", Version: "0.1.1"}, "cutter-0.1.1.tgz", "http://example.com/charts", "sha256:1234567890abc") i.Add(&chart.Metadata{Name: "cutter", Version: "0.1.0"}, "cutter-0.1.0.tgz", "http://example.com/charts", "sha256:1234567890abc") @@ -67,7 +67,7 @@ func TestLoadIndex(t *testing.T) { if err != nil { t.Fatal(err) } - i, err := LoadIndex(b) + i, err := loadIndex(b) if err != nil { t.Fatal(err) } @@ -75,7 +75,7 @@ func TestLoadIndex(t *testing.T) { } func TestLoadIndexFile(t *testing.T) { - i, err := LoadIndexFile(testfile) + i, err := NewChartRepositoryIndexFromFile(testfile) if err != nil { t.Fatal(err) } @@ -83,13 +83,13 @@ func TestLoadIndexFile(t *testing.T) { } func TestMerge(t *testing.T) { - ind1 := NewIndexFile() + ind1 := NewChartRepositoryIndex() ind1.Add(&chart.Metadata{ Name: "dreadnought", Version: "0.1.0", }, "dreadnought-0.1.0.tgz", "http://example.com", "aaaa") - ind2 := NewIndexFile() + ind2 := NewChartRepositoryIndex() ind2.Add(&chart.Metadata{ Name: "dreadnought", Version: "0.2.0", @@ -132,21 +132,30 @@ func TestDownloadIndexFile(t *testing.T) { } defer os.RemoveAll(dirName) - path := filepath.Join(dirName, testRepo+"-index.yaml") - if err := DownloadIndexFile(testRepo, srv.URL, path); err != nil { + indexFilePath := filepath.Join(dirName, testRepo+"-index.yaml") + r, err := NewChartRepository(&ChartRepositoryConfig{ + Name: testRepo, + URL: srv.URL, + Cache: indexFilePath, + }) + if err != nil { + t.Errorf("Problem creating chart repository from %s: %v", testRepo, err) + } + + if err := r.DownloadIndexFile(); err != nil { t.Errorf("%#v", err) } - if _, err := os.Stat(path); err != nil { + if _, err := os.Stat(indexFilePath); err != nil { t.Errorf("error finding created index file: %#v", err) } - b, err := ioutil.ReadFile(path) + b, err := ioutil.ReadFile(indexFilePath) if err != nil { t.Errorf("error reading index file: %#v", err) } - i, err := LoadIndex(b) + i, err := loadIndex(b) if err != nil { t.Errorf("Index %q failed to parse: %s", testfile, err) return @@ -155,7 +164,7 @@ func TestDownloadIndexFile(t *testing.T) { verifyLocalIndex(t, i) } -func verifyLocalIndex(t *testing.T, i *IndexFile) { +func verifyLocalIndex(t *testing.T, i *ChartRepositoryIndex) { numEntries := len(i.Entries) if numEntries != 2 { t.Errorf("Expected 2 entries in index file but got %d", numEntries) @@ -255,7 +264,7 @@ func verifyLocalIndex(t *testing.T, i *IndexFile) { func TestIndexDirectory(t *testing.T) { dir := "testdata/repository" - index, err := IndexDirectory(dir, "http://localhost:8080") + index, err := NewChartRepositoryIndexFromDirectory(dir, "http://localhost:8080") if err != nil { t.Fatal(err) } @@ -305,8 +314,7 @@ func TestLoadUnversionedIndex(t *testing.T) { } func TestIndexAdd(t *testing.T) { - - i := NewIndexFile() + i := NewChartRepositoryIndex() i.Add(&chart.Metadata{Name: "clipper", Version: "0.1.0"}, "clipper-0.1.0.tgz", "http://example.com/charts", "sha256:1234567890") if i.Entries["clipper"][0].URLs[0] != "http://example.com/charts/clipper-0.1.0.tgz" { @@ -325,24 +333,3 @@ func TestIndexAdd(t *testing.T) { t.Errorf("Expected http://example.com/charts/deis-0.1.0.tgz, got %s", i.Entries["deis"][0].URLs[0]) } } - -func TestUrlJoin(t *testing.T) { - tests := []struct { - name, url, expect string - paths []string - }{ - {name: "URL, one path", url: "http://example.com", paths: []string{"hello"}, expect: "http://example.com/hello"}, - {name: "Long URL, one path", url: "http://example.com/but/first", paths: []string{"slurm"}, expect: "http://example.com/but/first/slurm"}, - {name: "URL, two paths", url: "http://example.com", paths: []string{"hello", "world"}, expect: "http://example.com/hello/world"}, - {name: "URL, no paths", url: "http://example.com", paths: []string{}, expect: "http://example.com"}, - {name: "basepath, two paths", url: "../example.com", paths: []string{"hello", "world"}, expect: "../example.com/hello/world"}, - } - - for _, tt := range tests { - if got, err := urlJoin(tt.url, tt.paths...); err != nil { - t.Errorf("%s: error %q", tt.name, err) - } else if got != tt.expect { - t.Errorf("%s: expected %q, got %q", tt.name, tt.expect, got) - } - } -} diff --git a/pkg/repo/local.go b/pkg/repo/local.go index f13a4d0ac..932811be2 100644 --- a/pkg/repo/local.go +++ b/pkg/repo/local.go @@ -82,7 +82,7 @@ func (s *RepositoryServer) htmlIndex(w http.ResponseWriter, r *http.Request) { t := htemplate.Must(htemplate.New("index.html").Parse(indexHTMLTemplate)) // load index lrp := filepath.Join(s.RepoPath, "index.yaml") - i, err := LoadIndexFile(lrp) + i, err := NewChartRepositoryIndexFromFile(lrp) if err != nil { http.Error(w, err.Error(), 500) return @@ -107,7 +107,7 @@ func AddChartToLocalRepo(ch *chart.Chart, path string) error { // Reindex adds an entry to the index file at the given path func Reindex(ch *chart.Chart, path string) error { name := ch.Metadata.Name + "-" + ch.Metadata.Version - y, err := LoadIndexFile(path) + y, err := NewChartRepositoryIndexFromFile(path) if err != nil { return err } diff --git a/pkg/repo/repo.go b/pkg/repo/repo.go index d7ef9f6a6..7d4a91387 100644 --- a/pkg/repo/repo.go +++ b/pkg/repo/repo.go @@ -21,64 +21,44 @@ import ( "fmt" "io/ioutil" "os" - "path/filepath" - "strings" "time" "github.com/ghodss/yaml" - - "k8s.io/helm/pkg/chartutil" - "k8s.io/helm/pkg/provenance" ) // ErrRepoOutOfDate indicates that the repository file is out of date, but // is fixable. var ErrRepoOutOfDate = errors.New("repository file is out of date") -// ChartRepository represents a chart repository -type ChartRepository struct { - RootPath string - URL string // URL of repository - ChartPaths []string - IndexFile *IndexFile -} - -// Entry represents one repo entry in a repositories listing. -type Entry struct { - Name string `json:"name"` - Cache string `json:"cache"` - URL string `json:"url"` -} - -// RepoFile represents the repositories.yaml file in $HELM_HOME -type RepoFile struct { - APIVersion string `json:"apiVersion"` - Generated time.Time `json:"generated"` - Repositories []*Entry `json:"repositories"` +// RepositoryFile represents the repositories.yaml file in $HELM_HOME +type RepositoryFile struct { + APIVersion string `json:"apiVersion"` + Generated time.Time `json:"generated"` + Repositories []*ChartRepositoryConfig `json:"repositories"` } -// NewRepoFile generates an empty repositories file. +// NewRepositoryFile generates an empty repositories file. // // Generated and APIVersion are automatically set. -func NewRepoFile() *RepoFile { - return &RepoFile{ +func NewRepositoryFile() *RepositoryFile { + return &RepositoryFile{ APIVersion: APIVersionV1, Generated: time.Now(), - Repositories: []*Entry{}, + Repositories: []*ChartRepositoryConfig{}, } } -// LoadRepositoriesFile takes a file at the given path and returns a RepoFile object +// LoadRepositoriesFile takes a file at the given path and returns a RepositoryFile object // -// If this returns ErrRepoOutOfDate, it also returns a recovered RepoFile that +// If this returns ErrRepoOutOfDate, it also returns a recovered RepositoryFile that // can be saved as a replacement to the out of date file. -func LoadRepositoriesFile(path string) (*RepoFile, error) { +func LoadRepositoriesFile(path string) (*RepositoryFile, error) { b, err := ioutil.ReadFile(path) if err != nil { return nil, err } - r := &RepoFile{} + r := &RepositoryFile{} err = yaml.Unmarshal(b, r) if err != nil { return nil, err @@ -90,9 +70,9 @@ func LoadRepositoriesFile(path string) (*RepoFile, error) { if err = yaml.Unmarshal(b, &m); err != nil { return nil, err } - r := NewRepoFile() + r := NewRepositoryFile() for k, v := range m { - r.Add(&Entry{ + r.Add(&ChartRepositoryConfig{ Name: k, URL: v, Cache: fmt.Sprintf("%s-index.yaml", k), @@ -105,13 +85,13 @@ func LoadRepositoriesFile(path string) (*RepoFile, error) { } // Add adds one or more repo entries to a repo file. -func (r *RepoFile) Add(re ...*Entry) { +func (r *RepositoryFile) Add(re ...*ChartRepositoryConfig) { r.Repositories = append(r.Repositories, re...) } // Update attempts to replace one or more repo entries in a repo file. If an // entry with the same name doesn't exist in the repo file it will add it. -func (r *RepoFile) Update(re ...*Entry) { +func (r *RepositoryFile) Update(re ...*ChartRepositoryConfig) { for _, target := range re { found := false for j, repo := range r.Repositories { @@ -128,7 +108,7 @@ func (r *RepoFile) Update(re ...*Entry) { } // Has returns true if the given name is already a repository name. -func (r *RepoFile) Has(name string) bool { +func (r *RepositoryFile) Has(name string) bool { for _, rf := range r.Repositories { if rf.Name == name { return true @@ -138,8 +118,8 @@ func (r *RepoFile) Has(name string) bool { } // Remove removes the entry from the list of repositories. -func (r *RepoFile) Remove(name string) bool { - cp := []*Entry{} +func (r *RepositoryFile) Remove(name string) bool { + cp := []*ChartRepositoryConfig{} found := false for _, rf := range r.Repositories { if rf.Name == name { @@ -153,90 +133,10 @@ func (r *RepoFile) Remove(name string) bool { } // WriteFile writes a repositories file to the given path. -func (r *RepoFile) WriteFile(path string, perm os.FileMode) error { +func (r *RepositoryFile) WriteFile(path string, perm os.FileMode) error { data, err := yaml.Marshal(r) if err != nil { return err } return ioutil.WriteFile(path, data, perm) } - -// LoadChartRepository loads a directory of charts as if it were a repository. -// -// It requires the presence of an index.yaml file in the directory. -// -// This function evaluates the contents of the directory and -// returns a ChartRepository -func LoadChartRepository(dir, url string) (*ChartRepository, error) { - dirInfo, err := os.Stat(dir) - if err != nil { - return nil, err - } - - if !dirInfo.IsDir() { - return nil, fmt.Errorf("%q is not a directory", dir) - } - - r := &ChartRepository{RootPath: dir, URL: url} - - // FIXME: Why are we recursively walking directories? - // FIXME: Why are we not reading the repositories.yaml to figure out - // what repos to use? - filepath.Walk(dir, func(path string, f os.FileInfo, err error) error { - if !f.IsDir() { - if strings.Contains(f.Name(), "-index.yaml") { - i, err := LoadIndexFile(path) - if err != nil { - return nil - } - r.IndexFile = i - } else if strings.HasSuffix(f.Name(), ".tgz") { - r.ChartPaths = append(r.ChartPaths, path) - } - } - return nil - }) - return r, nil -} - -func (r *ChartRepository) saveIndexFile() error { - index, err := yaml.Marshal(r.IndexFile) - if err != nil { - return err - } - return ioutil.WriteFile(filepath.Join(r.RootPath, indexPath), index, 0644) -} - -// Index generates an index for the chart repository and writes an index.yaml file. -func (r *ChartRepository) Index() error { - err := r.generateIndex() - if err != nil { - return err - } - return r.saveIndexFile() -} - -func (r *ChartRepository) generateIndex() error { - if r.IndexFile == nil { - r.IndexFile = NewIndexFile() - } - - for _, path := range r.ChartPaths { - ch, err := chartutil.Load(path) - if err != nil { - return err - } - - digest, err := provenance.DigestFile(path) - if err != nil { - return err - } - - if !r.IndexFile.Has(ch.Metadata.Name, ch.Metadata.Version) { - r.IndexFile.Add(ch.Metadata, path, r.URL, digest) - } - // TODO: If a chart exists, but has a different Digest, should we error? - } - r.IndexFile.SortEntries() - return nil -} diff --git a/pkg/repo/repo_test.go b/pkg/repo/repo_test.go index 704c3fe3d..6ee733fcc 100644 --- a/pkg/repo/repo_test.go +++ b/pkg/repo/repo_test.go @@ -16,29 +16,19 @@ limitations under the License. package repo -import ( - "os" - "path/filepath" - "reflect" - "testing" - "time" - - "k8s.io/helm/pkg/proto/hapi/chart" -) +import "testing" const testRepositoriesFile = "testdata/repositories.yaml" -const testRepository = "testdata/repository" -const testURL = "http://example-charts.com" func TestRepoFile(t *testing.T) { - rf := NewRepoFile() + rf := NewRepositoryFile() rf.Add( - &Entry{ + &ChartRepositoryConfig{ Name: "stable", URL: "https://example.com/stable/charts", Cache: "stable-index.yaml", }, - &Entry{ + &ChartRepositoryConfig{ Name: "incubator", URL: "https://example.com/incubator", Cache: "incubator-index.yaml", @@ -68,15 +58,15 @@ func TestRepoFile(t *testing.T) { } } -func TestLoadRepositoriesFile(t *testing.T) { - expects := NewRepoFile() +func TestNewRepositoriesFile(t *testing.T) { + expects := NewRepositoryFile() expects.Add( - &Entry{ + &ChartRepositoryConfig{ Name: "stable", URL: "https://example.com/stable/charts", Cache: "stable-index.yaml", }, - &Entry{ + &ChartRepositoryConfig{ Name: "incubator", URL: "https://example.com/incubator", Cache: "incubator-index.yaml", @@ -106,7 +96,7 @@ func TestLoadRepositoriesFile(t *testing.T) { } } -func TestLoadPreV1RepositoriesFile(t *testing.T) { +func TestNewPreV1RepositoriesFile(t *testing.T) { r, err := LoadRepositoriesFile("testdata/old-repositories.yaml") if err != nil && err != ErrRepoOutOfDate { t.Fatal(err) @@ -126,139 +116,3 @@ func TestLoadPreV1RepositoriesFile(t *testing.T) { t.Errorf("expected the best charts ever. Got %#v", r.Repositories) } } - -func TestLoadChartRepository(t *testing.T) { - cr, err := LoadChartRepository(testRepository, testURL) - if err != nil { - t.Errorf("Problem loading chart repository from %s: %v", testRepository, err) - } - - paths := []string{filepath.Join(testRepository, "frobnitz-1.2.3.tgz"), filepath.Join(testRepository, "sprocket-1.1.0.tgz"), filepath.Join(testRepository, "sprocket-1.2.0.tgz")} - - if cr.RootPath != testRepository { - t.Errorf("Expected %s as RootPath but got %s", testRepository, cr.RootPath) - } - - if !reflect.DeepEqual(cr.ChartPaths, paths) { - t.Errorf("Expected %#v but got %#v\n", paths, cr.ChartPaths) - } - - if cr.URL != testURL { - t.Errorf("Expected url for chart repository to be %s but got %s", testURL, cr.URL) - } -} - -func TestIndex(t *testing.T) { - cr, err := LoadChartRepository(testRepository, testURL) - if err != nil { - t.Errorf("Problem loading chart repository from %s: %v", testRepository, err) - } - - err = cr.Index() - if err != nil { - t.Errorf("Error performing index: %v\n", err) - } - - tempIndexPath := filepath.Join(testRepository, indexPath) - actual, err := LoadIndexFile(tempIndexPath) - defer os.Remove(tempIndexPath) // clean up - if err != nil { - t.Errorf("Error loading index file %v", err) - } - verifyIndex(t, actual) - - // Re-index and test again. - err = cr.Index() - if err != nil { - t.Errorf("Error performing re-index: %s\n", err) - } - second, err := LoadIndexFile(tempIndexPath) - if err != nil { - t.Errorf("Error re-loading index file %v", err) - } - verifyIndex(t, second) -} - -func verifyIndex(t *testing.T, actual *IndexFile) { - - var empty time.Time - if actual.Generated == empty { - t.Errorf("Generated should be greater than 0: %s", actual.Generated) - } - - if actual.APIVersion != APIVersionV1 { - t.Error("Expected v1 API") - } - - entries := actual.Entries - if numEntries := len(entries); numEntries != 2 { - t.Errorf("Expected 2 charts to be listed in index file but got %v", numEntries) - } - - expects := map[string]ChartVersions{ - "frobnitz": { - { - Metadata: &chart.Metadata{ - Name: "frobnitz", - Version: "1.2.3", - }, - }, - }, - "sprocket": { - { - Metadata: &chart.Metadata{ - Name: "sprocket", - Version: "1.2.0", - }, - }, - { - Metadata: &chart.Metadata{ - Name: "sprocket", - Version: "1.1.0", - }, - }, - }, - } - - for name, versions := range expects { - got, ok := entries[name] - if !ok { - t.Errorf("Could not find %q entry", name) - continue - } - if len(versions) != len(got) { - t.Errorf("Expected %d versions, got %d", len(versions), len(got)) - continue - } - for i, e := range versions { - g := got[i] - if e.Name != g.Name { - t.Errorf("Expected %q, got %q", e.Name, g.Name) - } - if e.Version != g.Version { - t.Errorf("Expected %q, got %q", e.Version, g.Version) - } - if len(g.Keywords) != 3 { - t.Error("Expected 3 keyrwords.") - } - if len(g.Maintainers) != 2 { - t.Error("Expected 2 maintainers.") - } - if g.Created == empty { - t.Error("Expected created to be non-empty") - } - if g.Description == "" { - t.Error("Expected description to be non-empty") - } - if g.Home == "" { - t.Error("Expected home to be non-empty") - } - if g.Digest == "" { - t.Error("Expected digest to be non-empty") - } - if len(g.URLs) != 1 { - t.Error("Expected exactly 1 URL") - } - } - } -} diff --git a/pkg/repo/repotest/server.go b/pkg/repo/repotest/server.go index 0094a17fa..07973a692 100644 --- a/pkg/repo/repotest/server.go +++ b/pkg/repo/repotest/server.go @@ -24,6 +24,7 @@ import ( "github.com/ghodss/yaml" + "k8s.io/helm/cmd/helm/helmpath" "k8s.io/helm/pkg/repo" ) @@ -69,7 +70,7 @@ func NewServer(docroot string) *Server { } srv.start() // Add the testing repository as the only repo. - if err := setTestingRepository(docroot, "test", srv.URL()); err != nil { + if err := setTestingRepository(helmpath.Home(docroot), "test", srv.URL()); err != nil { panic(err) } return srv @@ -113,7 +114,7 @@ func (s *Server) CopyCharts(origin string) ([]string, error) { // CreateIndex will read docroot and generate an index.yaml file. func (s *Server) CreateIndex() error { // generate the index - index, err := repo.IndexDirectory(s.docroot, s.URL()) + index, err := repo.NewChartRepositoryIndexFromDirectory(s.docroot, s.URL()) if err != nil { return err } @@ -158,11 +159,13 @@ func (s *Server) LinkIndices() error { } // setTestingRepository sets up a testing repository.yaml with only the given name/URL. -func setTestingRepository(helmhome, name, url string) error { - rf := repo.NewRepoFile() - rf.Add(&repo.Entry{Name: name, URL: url}) - os.MkdirAll(filepath.Join(helmhome, "repository", name), 0755) - dest := filepath.Join(helmhome, "repository/repositories.yaml") - - return rf.WriteFile(dest, 0644) +func setTestingRepository(home helmpath.Home, name, url string) error { + r := repo.NewRepositoryFile() + r.Add(&repo.ChartRepositoryConfig{ + Name: name, + URL: url, + Cache: home.CacheIndex(name), + }) + os.MkdirAll(filepath.Join(home.Repository(), name), 0755) + return r.WriteFile(home.RepositoryFile(), 0644) } diff --git a/pkg/repo/repotest/server_test.go b/pkg/repo/repotest/server_test.go index 1d4c78e41..99917fc3b 100644 --- a/pkg/repo/repotest/server_test.go +++ b/pkg/repo/repotest/server_test.go @@ -77,7 +77,7 @@ func TestServer(t *testing.T) { return } - m := repo.NewIndexFile() + m := repo.NewChartRepositoryIndex() if err := yaml.Unmarshal(data, m); err != nil { t.Error(err) return diff --git a/pkg/tlsutil/tls.go b/pkg/tlsutil/tls.go new file mode 100644 index 000000000..05a671211 --- /dev/null +++ b/pkg/tlsutil/tls.go @@ -0,0 +1,68 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tlsutil + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" +) + +// NewClientTLS returns tls.Config appropriate for client auth. +func NewClientTLS(certFile, keyFile, caFile string) (*tls.Config, error) { + cert, err := CertFromFilePair(certFile, keyFile) + if err != nil { + return nil, err + } + cp, err := CertPoolFromFile(caFile) + if err != nil { + return nil, err + } + return &tls.Config{ + Certificates: []tls.Certificate{*cert}, + RootCAs: cp, + }, nil +} + +// CertPoolFromFile returns an x509.CertPool containing the certificates +// in the given PEM-encoded file. +// Returns an error if the file could not be read, a certificate could not +// be parsed, or if the file does not contain any certificates +func CertPoolFromFile(filename string) (*x509.CertPool, error) { + b, err := ioutil.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("can't read CA file: %v", filename) + } + cp := x509.NewCertPool() + if !cp.AppendCertsFromPEM(b) { + return nil, fmt.Errorf("failed to append certificates from file: %s", filename) + } + return cp, nil +} + +// CertFromFilePair returns an tls.Certificate containing the +// certificates public/private key pair from a pair of given PEM-encoded files. +// Returns an error if the file could not be read, a certificate could not +// be parsed, or if the file does not contain any certificates +func CertFromFilePair(certFile, keyFile string) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("can't load key pair from cert %s and key %s", certFile, keyFile) + } + return &cert, err +} diff --git a/pkg/urlutil/urlutil.go b/pkg/urlutil/urlutil.go new file mode 100644 index 000000000..8026fd245 --- /dev/null +++ b/pkg/urlutil/urlutil.go @@ -0,0 +1,66 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package urlutil + +import ( + "net/url" + "path" + "path/filepath" +) + +// URLJoin joins a base URL to one or more path components. +// +// It's like filepath.Join for URLs. If the baseURL is pathish, this will still +// perform a join. +// +// If the URL is unparsable, this returns an error. +func URLJoin(baseURL string, paths ...string) (string, error) { + u, err := url.Parse(baseURL) + if err != nil { + return "", err + } + // We want path instead of filepath because path always uses /. + all := []string{u.Path} + all = append(all, paths...) + u.Path = path.Join(all...) + return u.String(), nil +} + +// URLAreEqual normalizes two URLs and then compares for equality. +// +// TODO: This and the urlJoin functions should really be moved to a 'urlutil' package. +func URLAreEqual(a, b string) bool { + au, err := url.Parse(a) + if err != nil { + a = filepath.Clean(a) + b = filepath.Clean(b) + // If urls are paths, return true only if they are an exact match + return a == b + } + bu, err := url.Parse(b) + if err != nil { + return false + } + + for _, u := range []*url.URL{au, bu} { + if u.Path == "" { + u.Path = "/" + } + u.Path = filepath.Clean(u.Path) + } + return au.String() == bu.String() +} diff --git a/pkg/urlutil/urlutil_test.go b/pkg/urlutil/urlutil_test.go new file mode 100644 index 000000000..d4aa1139d --- /dev/null +++ b/pkg/urlutil/urlutil_test.go @@ -0,0 +1,64 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package urlutil + +import "testing" + +func TestUrlJoin(t *testing.T) { + tests := []struct { + name, url, expect string + paths []string + }{ + {name: "URL, one path", url: "http://example.com", paths: []string{"hello"}, expect: "http://example.com/hello"}, + {name: "Long URL, one path", url: "http://example.com/but/first", paths: []string{"slurm"}, expect: "http://example.com/but/first/slurm"}, + {name: "URL, two paths", url: "http://example.com", paths: []string{"hello", "world"}, expect: "http://example.com/hello/world"}, + {name: "URL, no paths", url: "http://example.com", paths: []string{}, expect: "http://example.com"}, + {name: "basepath, two paths", url: "../example.com", paths: []string{"hello", "world"}, expect: "../example.com/hello/world"}, + } + + for _, tt := range tests { + if got, err := URLJoin(tt.url, tt.paths...); err != nil { + t.Errorf("%s: error %q", tt.name, err) + } else if got != tt.expect { + t.Errorf("%s: expected %q, got %q", tt.name, tt.expect, got) + } + } +} + +func TestUrlAreEqual(t *testing.T) { + for _, tt := range []struct { + a, b string + match bool + }{ + {"http://example.com", "http://example.com", true}, + {"http://example.com", "http://another.example.com", false}, + {"https://example.com", "https://example.com", true}, + {"http://example.com/", "http://example.com", true}, + {"https://example.com", "http://example.com", false}, + {"http://example.com/foo", "http://example.com/foo/", true}, + {"http://example.com/foo//", "http://example.com/foo/", true}, + {"http://example.com/./foo/", "http://example.com/foo/", true}, + {"http://example.com/bar/../foo/", "http://example.com/foo/", true}, + {"/foo", "/foo", true}, + {"/foo", "/foo/", true}, + {"/foo/.", "/foo/", true}, + } { + if tt.match != URLAreEqual(tt.a, tt.b) { + t.Errorf("Expected %q==%q to be %t", tt.a, tt.b, tt.match) + } + } +} From 89ab7555dbf1e1f2817aeb1b4f65ca7a52aaf86d Mon Sep 17 00:00:00 2001 From: Anton Galitsyn Date: Tue, 27 Dec 2016 23:47:27 +0700 Subject: [PATCH 2/7] Get http.Client from referenced repository --- cmd/helm/downloader/chart_downloader.go | 118 +++++++++++------- cmd/helm/downloader/chart_downloader_test.go | 68 +++++++--- cmd/helm/downloader/manager.go | 8 +- .../cache/testing-basicauth-index.yaml | 15 +++ .../repository/cache/testing-https-index.yaml | 15 +++ .../repository/cache/testing-index.yaml | 13 ++ .../helmhome/repository/repositories.yaml | 4 + cmd/helm/helm_test.go | 2 +- cmd/helm/init.go | 4 +- cmd/helm/repo_add.go | 2 +- cmd/helm/repo_add_test.go | 2 +- cmd/helm/repo_list.go | 2 +- cmd/helm/repo_remove.go | 2 +- cmd/helm/repo_remove_test.go | 2 +- cmd/helm/repo_update.go | 2 +- cmd/helm/search.go | 2 +- pkg/repo/chartrepo.go | 8 ++ pkg/repo/repo.go | 4 +- pkg/repo/repo_test.go | 4 +- pkg/urlutil/urlutil.go | 15 +++ 20 files changed, 215 insertions(+), 77 deletions(-) create mode 100644 cmd/helm/downloader/testdata/helmhome/repository/cache/testing-basicauth-index.yaml create mode 100644 cmd/helm/downloader/testdata/helmhome/repository/cache/testing-https-index.yaml diff --git a/cmd/helm/downloader/chart_downloader.go b/cmd/helm/downloader/chart_downloader.go index bb64ff7fc..57e79e9c8 100644 --- a/cmd/helm/downloader/chart_downloader.go +++ b/cmd/helm/downloader/chart_downloader.go @@ -76,12 +76,12 @@ type ChartDownloader struct { // Returns a string path to the location where the file was downloaded and a verification // (if provenance was verified), or an error if something bad happened. func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *provenance.Verification, error) { - // resolve URL - u, err := c.ResolveChartVersion(ref, version) + u, client, err := c.ResolveChartVersion(ref, version) if err != nil { return "", nil, err } - data, err := download(u.String()) + + data, err := download(u.String(), client) if err != nil { return "", nil, err } @@ -95,8 +95,7 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven // If provenance is requested, verify it. ver := &provenance.Verification{} if c.Verify > VerifyNever { - - body, err := download(u.String() + ".prov") + body, err := download(u.String()+".prov", client) if err != nil { if c.Verify == VerifyAlways { return destfile, ver, fmt.Errorf("Failed to fetch provenance %q", u.String()+".prov") @@ -132,63 +131,75 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven // * If version is non-empty, this will return the URL for that version // * If version is empty, this will return the URL for the latest version // * If no version can be found, an error is returned -func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, error) { - // See if it's already a full URL. - // FIXME: Why do we use url.ParseRequestURI instead of url.Parse? - u, err := url.ParseRequestURI(ref) - if err == nil { - // If it has a scheme and host and path, it's a full URL - if u.IsAbs() && len(u.Host) > 0 && len(u.Path) > 0 { - return u, nil - } - return u, fmt.Errorf("invalid chart url format: %s", ref) +func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, *http.Client, error) { + u, err := url.Parse(ref) + if err != nil { + return nil, nil, fmt.Errorf("invalid chart URL format: %s", ref) } - r, err := repo.LoadRepositoriesFile(c.HelmHome.RepositoryFile()) + rf, err := repo.LoadRepositoryFile(c.HelmHome.RepositoryFile()) if err != nil { - return u, err + return nil, nil, err } - // See if it's of the form: repo/path_to_chart - p := strings.SplitN(ref, "/", 2) - if len(p) < 2 { - return u, fmt.Errorf("invalid chart url format: %s", ref) + var ( + chartName string + rc *repo.ChartRepositoryConfig + ) + if u.IsAbs() && len(u.Host) > 0 && len(u.Path) > 0 { + // If it has a scheme and host and path, it's a full URL + p := strings.SplitN(strings.TrimLeft(u.Path, "/"), "-", 2) + if len(p) < 2 { + return nil, nil, fmt.Errorf("Seems that chart path is not in form of repo_url/path_to_chart, got: %s", u) + } + chartName = p[0] + u.Path = "" + rc, err = pickChartRepositoryConfigByURL(u.String(), rf.Repositories) + if err != nil { + return nil, nil, err + } + } else { + // See if it's of the form: repo/path_to_chart + p := strings.SplitN(u.Path, "/", 2) + if len(p) < 2 { + return nil, nil, fmt.Errorf("Non-absolute URLs should be in form of repo_name/path_to_chart, got: %s", u) + } + + repoName := p[0] + chartName = p[1] + rc, err = pickChartRepositoryConfigByName(repoName, rf.Repositories) + if err != nil { + return nil, nil, err + } } - repoName := p[0] - chartName := p[1] - rf, err := findRepoEntry(repoName, r.Repositories) + r, err := repo.NewChartRepository(rc) if err != nil { - return u, err - } - if rf.URL == "" { - return u, fmt.Errorf("no URL found for repository %q", repoName) + return nil, nil, err } // Next, we need to load the index, and actually look up the chart. - i, err := repo.NewChartRepositoryIndexFromFile(c.HelmHome.CacheIndex(repoName)) + i, err := repo.NewChartRepositoryIndexFromFile(c.HelmHome.CacheIndex(r.Config.Name)) if err != nil { - return u, fmt.Errorf("no cached repo found. (try 'helm repo update'). %s", err) + return nil, nil, fmt.Errorf("no cached repo found. (try 'helm repo update'). %s", err) } cv, err := i.Get(chartName, version) if err != nil { - return u, fmt.Errorf("chart %q not found in %s index. (try 'helm repo update'). %s", chartName, repoName, err) + return nil, nil, fmt.Errorf("chart %q not found in %s index. (try 'helm repo update'). %s", chartName, r.Config.Name, err) } if len(cv.URLs) == 0 { - return u, fmt.Errorf("chart %q has no downloadable URLs", ref) + return nil, nil, fmt.Errorf("chart %q has no downloadable URLs", ref) } - return url.Parse(cv.URLs[0]) -} -func findRepoEntry(name string, repos []*repo.ChartRepositoryConfig) (*repo.ChartRepositoryConfig, error) { - for _, re := range repos { - if re.Name == name { - return re, nil - } + // TODO: Seems that picking first URL is not fully correct + u, err = url.Parse(cv.URLs[0]) + if err != nil { + return nil, nil, fmt.Errorf("invalid chart URL format: %s", ref) } - return nil, fmt.Errorf("no repo named %q", name) + + return u, r.Client, nil } // VerifyChart takes a path to a chart archive and a keyring, and verifies the chart. @@ -217,11 +228,11 @@ func VerifyChart(path string, keyring string) (*provenance.Verification, error) return sig.Verify(path, provfile) } -// download performs a simple HTTP Get and returns the body. -func download(href string) (*bytes.Buffer, error) { +// download performs a HTTP Get using specified client and returns the body. +func download(href string, client *http.Client) (*bytes.Buffer, error) { buf := bytes.NewBuffer(nil) - resp, err := http.Get(href) + resp, err := client.Get(href) if err != nil { return buf, err } @@ -241,3 +252,24 @@ func download(href string) (*bytes.Buffer, error) { func isTar(filename string) bool { return strings.ToLower(filepath.Ext(filename)) == ".tgz" } + +func pickChartRepositoryConfigByName(name string, cfgs []*repo.ChartRepositoryConfig) (*repo.ChartRepositoryConfig, error) { + for _, rc := range cfgs { + if rc.Name == name { + if rc.URL == "" { + return nil, fmt.Errorf("no URL found for repository %s", name) + } + return rc, nil + } + } + return nil, fmt.Errorf("repo %s not found", name) +} + +func pickChartRepositoryConfigByURL(u string, cfgs []*repo.ChartRepositoryConfig) (*repo.ChartRepositoryConfig, error) { + for _, rc := range cfgs { + if rc.URL == u { + return rc, nil + } + } + return nil, fmt.Errorf("repo with URL %s not found", u) +} diff --git a/cmd/helm/downloader/chart_downloader_test.go b/cmd/helm/downloader/chart_downloader_test.go index c3d06f1d6..7a7fbe76f 100644 --- a/cmd/helm/downloader/chart_downloader_test.go +++ b/cmd/helm/downloader/chart_downloader_test.go @@ -37,9 +37,9 @@ func TestResolveChartRef(t *testing.T) { {name: "full URL", ref: "http://example.com/foo-1.2.3.tgz", expect: "http://example.com/foo-1.2.3.tgz"}, {name: "full URL, HTTPS", ref: "https://example.com/foo-1.2.3.tgz", expect: "https://example.com/foo-1.2.3.tgz"}, {name: "full URL, with authentication", ref: "http://username:password@example.com/foo-1.2.3.tgz", expect: "http://username:password@example.com/foo-1.2.3.tgz"}, - {name: "full URL, HTTPS, irrelevant version", ref: "https://example.com/foo-1.2.3.tgz", version: "0.1.0", expect: "https://example.com/foo-1.2.3.tgz"}, {name: "reference, testing repo", ref: "testing/alpine", expect: "http://example.com/alpine-1.2.3.tgz"}, {name: "reference, version, testing repo", ref: "testing/alpine", version: "0.2.0", expect: "http://example.com/alpine-0.2.0.tgz"}, + {name: "full URL, HTTPS, irrelevant version", ref: "https://example.com/foo-1.2.3.tgz", version: "0.1.0", expect: "https://example.com/foo-1.2.3.tgz", fail: true}, {name: "full URL, file", ref: "file:///foo-1.2.3.tgz", fail: true}, {name: "invalid", ref: "invalid-1.2.3", fail: true}, {name: "not found", ref: "nosuchthing/invalid-1.2.3", fail: true}, @@ -51,7 +51,7 @@ func TestResolveChartRef(t *testing.T) { } for _, tt := range tests { - u, err := c.ResolveChartVersion(tt.ref, tt.version) + u, _, err := c.ResolveChartVersion(tt.ref, tt.version) if err != nil { if tt.fail { continue @@ -84,7 +84,7 @@ func TestDownload(t *testing.T) { })) defer srv.Close() - got, err := download(srv.URL) + got, err := download(srv.URL, http.DefaultClient) if err != nil { t.Fatal(err) } @@ -105,7 +105,7 @@ func TestDownload(t *testing.T) { u, _ := url.ParseRequestURI(basicAuthSrv.URL) u.User = url.UserPassword("username", "password") - got, err = download(u.String()) + got, err = download(u.String(), http.DefaultClient) if err != nil { t.Fatal(err) } @@ -133,25 +133,43 @@ func TestIsTar(t *testing.T) { } func TestDownloadTo(t *testing.T) { - hh, err := ioutil.TempDir("", "helm-downloadto-") + tmp, err := ioutil.TempDir("", "helm-downloadto-") if err != nil { t.Fatal(err) } - defer os.RemoveAll(hh) + defer os.RemoveAll(tmp) - dest := filepath.Join(hh, "dest") - os.MkdirAll(dest, 0755) + hh := helmpath.Home(tmp) + dest := filepath.Join(hh.String(), "dest") + configDirectories := []string{ + hh.String(), + hh.Repository(), + hh.Cache(), + dest, + } + for _, p := range configDirectories { + if fi, err := os.Stat(p); err != nil { + if err := os.MkdirAll(p, 0755); err != nil { + t.Fatalf("Could not create %s: %s", p, err) + } + } else if !fi.IsDir() { + t.Fatalf("%s must be a directory", p) + } + } // Set up a fake repo - srv := repotest.NewServer(hh) + srv := repotest.NewServer(tmp) defer srv.Stop() if _, err := srv.CopyCharts("testdata/*.tgz*"); err != nil { t.Error(err) return } + if err := srv.LinkIndices(); err != nil { + t.Fatal(err) + } c := ChartDownloader{ - HelmHome: helmpath.Home("testdata/helmhome"), + HelmHome: hh, Out: os.Stderr, Verify: VerifyAlways, Keyring: "testdata/helm-test-key.pub", @@ -178,25 +196,43 @@ func TestDownloadTo(t *testing.T) { } func TestDownloadTo_VerifyLater(t *testing.T) { - hh, err := ioutil.TempDir("", "helm-downloadto-") + tmp, err := ioutil.TempDir("", "helm-downloadto-") if err != nil { t.Fatal(err) } - defer os.RemoveAll(hh) + defer os.RemoveAll(tmp) - dest := filepath.Join(hh, "dest") - os.MkdirAll(dest, 0755) + hh := helmpath.Home(tmp) + dest := filepath.Join(hh.String(), "dest") + configDirectories := []string{ + hh.String(), + hh.Repository(), + hh.Cache(), + dest, + } + for _, p := range configDirectories { + if fi, err := os.Stat(p); err != nil { + if err := os.MkdirAll(p, 0755); err != nil { + t.Fatalf("Could not create %s: %s", p, err) + } + } else if !fi.IsDir() { + t.Fatalf("%s must be a directory", p) + } + } // Set up a fake repo - srv := repotest.NewServer(hh) + srv := repotest.NewServer(tmp) defer srv.Stop() if _, err := srv.CopyCharts("testdata/*.tgz*"); err != nil { t.Error(err) return } + if err := srv.LinkIndices(); err != nil { + t.Fatal(err) + } c := ChartDownloader{ - HelmHome: helmpath.Home("testdata/helmhome"), + HelmHome: hh, Out: os.Stderr, Verify: VerifyLater, } diff --git a/cmd/helm/downloader/manager.go b/cmd/helm/downloader/manager.go index a64ed3bf3..daa65ef97 100644 --- a/cmd/helm/downloader/manager.go +++ b/cmd/helm/downloader/manager.go @@ -213,7 +213,7 @@ func (m *Manager) downloadAll(deps []*chartutil.Dependency) error { // hasAllRepos ensures that all of the referenced deps are in the local repo cache. func (m *Manager) hasAllRepos(deps []*chartutil.Dependency) error { - rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) + rf, err := repo.LoadRepositoryFile(m.HelmHome.RepositoryFile()) if err != nil { return err } @@ -244,7 +244,7 @@ func (m *Manager) hasAllRepos(deps []*chartutil.Dependency) error { // getRepoNames returns the repo names of the referenced deps which can be used to fetch the cahced index file. func (m *Manager) getRepoNames(deps []*chartutil.Dependency) (map[string]string, error) { - rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) + rf, err := repo.LoadRepositoryFile(m.HelmHome.RepositoryFile()) if err != nil { return nil, err } @@ -277,7 +277,7 @@ func (m *Manager) getRepoNames(deps []*chartutil.Dependency) (map[string]string, // UpdateRepositories updates all of the local repos to the latest. func (m *Manager) UpdateRepositories() error { - rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) + rf, err := repo.LoadRepositoryFile(m.HelmHome.RepositoryFile()) if err != nil { return err } @@ -409,7 +409,7 @@ func (m *Manager) loadChartRepositories() (map[string]*repo.ChartRepository, err repoyaml := m.HelmHome.RepositoryFile() // Load repositories.yaml file - rf, err := repo.LoadRepositoriesFile(repoyaml) + rf, err := repo.LoadRepositoryFile(repoyaml) if err != nil { return indices, fmt.Errorf("failed to load %s: %s", repoyaml, err) } diff --git a/cmd/helm/downloader/testdata/helmhome/repository/cache/testing-basicauth-index.yaml b/cmd/helm/downloader/testdata/helmhome/repository/cache/testing-basicauth-index.yaml new file mode 100644 index 000000000..47bb1b77c --- /dev/null +++ b/cmd/helm/downloader/testdata/helmhome/repository/cache/testing-basicauth-index.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +entries: + foo: + - name: foo + description: Foo Chart + engine: gotpl + home: https://k8s.io/helm + keywords: [] + maintainers: [] + sources: + - https://github.com/kubernetes/charts + urls: + - http://username:password@example.com/foo-1.2.3.tgz + version: 1.2.3 + checksum: 0e6661f193211d7a5206918d42f5c2a9470b737d diff --git a/cmd/helm/downloader/testdata/helmhome/repository/cache/testing-https-index.yaml b/cmd/helm/downloader/testdata/helmhome/repository/cache/testing-https-index.yaml new file mode 100644 index 000000000..872478c3f --- /dev/null +++ b/cmd/helm/downloader/testdata/helmhome/repository/cache/testing-https-index.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +entries: + foo: + - name: foo + description: Foo Chart + engine: gotpl + home: https://k8s.io/helm + keywords: [] + maintainers: [] + sources: + - https://github.com/kubernetes/charts + urls: + - https://example.com/foo-1.2.3.tgz + version: 1.2.3 + checksum: 0e6661f193211d7a5206918d42f5c2a9470b737d diff --git a/cmd/helm/downloader/testdata/helmhome/repository/cache/testing-index.yaml b/cmd/helm/downloader/testdata/helmhome/repository/cache/testing-index.yaml index cec18b069..14cdffece 100644 --- a/cmd/helm/downloader/testdata/helmhome/repository/cache/testing-index.yaml +++ b/cmd/helm/downloader/testdata/helmhome/repository/cache/testing-index.yaml @@ -28,3 +28,16 @@ entries: maintainers: [] engine: "" icon: "" + foo: + - name: foo + description: Foo Chart + engine: gotpl + home: https://k8s.io/helm + keywords: [] + maintainers: [] + sources: + - https://github.com/kubernetes/charts + urls: + - http://example.com/foo-1.2.3.tgz + version: 1.2.3 + checksum: 0e6661f193211d7a5206918d42f5c2a9470b737d diff --git a/cmd/helm/downloader/testdata/helmhome/repository/repositories.yaml b/cmd/helm/downloader/testdata/helmhome/repository/repositories.yaml index c7ddf316a..200f370bd 100644 --- a/cmd/helm/downloader/testdata/helmhome/repository/repositories.yaml +++ b/cmd/helm/downloader/testdata/helmhome/repository/repositories.yaml @@ -2,5 +2,9 @@ apiVersion: v1 repositories: - name: testing url: "http://example.com" + - name: testing-https + url: "https://example.com" + - name: testing-basicauth + url: "http://username:password@example.com" - name: kubernetes-charts url: "http://example.com/charts" diff --git a/cmd/helm/helm_test.go b/cmd/helm/helm_test.go index 83a086875..7dfc34dae 100644 --- a/cmd/helm/helm_test.go +++ b/cmd/helm/helm_test.go @@ -270,7 +270,7 @@ func ensureTestHome(home helmpath.Home, t *testing.T) error { } else if fi.IsDir() { return fmt.Errorf("%s must be a file, not a directory", repoFile) } - if r, err := repo.LoadRepositoriesFile(repoFile); err == repo.ErrRepoOutOfDate { + if r, err := repo.LoadRepositoryFile(repoFile); err == repo.ErrRepoOutOfDate { t.Log("Updating repository file format...") if err := r.WriteFile(repoFile, 0644); err != nil { return err diff --git a/cmd/helm/init.go b/cmd/helm/init.go index b46c32550..43a0ece81 100644 --- a/cmd/helm/init.go +++ b/cmd/helm/init.go @@ -55,7 +55,7 @@ To dump a manifest containing the Tiller deployment YAML, combine the const ( stableRepository = "stable" localRepository = "local" - stableRepositoryURL = "https://kubernetes-charts.storage.googleapis.com/" + stableRepositoryURL = "https://kubernetes-charts.storage.googleapis.com" // This is the IPv4 loopback, not localhost, because we have to force IPv4 // for Dockerized Helm: https://github.com/kubernetes/helm/issues/1410 localRepositoryURL = "http://127.0.0.1:8879/charts" @@ -236,7 +236,7 @@ func initLocalRepo(indexFile, cacheFile string) (*repo.ChartRepositoryConfig, er } func ensureRepoFileFormat(file string, out io.Writer) error { - r, err := repo.LoadRepositoriesFile(file) + r, err := repo.LoadRepositoryFile(file) if err == repo.ErrRepoOutOfDate { fmt.Fprintln(out, "Updating repository file format...") if err := r.WriteFile(file, 0644); err != nil { diff --git a/cmd/helm/repo_add.go b/cmd/helm/repo_add.go index 0abaa9740..2d9a62d1e 100644 --- a/cmd/helm/repo_add.go +++ b/cmd/helm/repo_add.go @@ -78,7 +78,7 @@ func (a *repoAddCmd) run() error { } func addRepository(name, url string, home helmpath.Home, certFile, keyFile, caFile string, noUpdate bool) error { - f, err := repo.LoadRepositoriesFile(home.RepositoryFile()) + f, err := repo.LoadRepositoryFile(home.RepositoryFile()) if err != nil { return err } diff --git a/cmd/helm/repo_add_test.go b/cmd/helm/repo_add_test.go index 7c5342177..b823777f7 100644 --- a/cmd/helm/repo_add_test.go +++ b/cmd/helm/repo_add_test.go @@ -84,7 +84,7 @@ func TestRepoAdd(t *testing.T) { t.Error(err) } - f, err := repo.LoadRepositoriesFile(hh.RepositoryFile()) + f, err := repo.LoadRepositoryFile(hh.RepositoryFile()) if err != nil { t.Error(err) } diff --git a/cmd/helm/repo_list.go b/cmd/helm/repo_list.go index a3816facd..a4f7edec5 100644 --- a/cmd/helm/repo_list.go +++ b/cmd/helm/repo_list.go @@ -51,7 +51,7 @@ func newRepoListCmd(out io.Writer) *cobra.Command { } func (a *repoListCmd) run() error { - f, err := repo.LoadRepositoriesFile(a.home.RepositoryFile()) + f, err := repo.LoadRepositoryFile(a.home.RepositoryFile()) if err != nil { return err } diff --git a/cmd/helm/repo_remove.go b/cmd/helm/repo_remove.go index f7671cab0..ad2a9c188 100644 --- a/cmd/helm/repo_remove.go +++ b/cmd/helm/repo_remove.go @@ -62,7 +62,7 @@ func (r *repoRemoveCmd) run() error { func removeRepoLine(out io.Writer, name string, home helmpath.Home) error { repoFile := home.RepositoryFile() - r, err := repo.LoadRepositoriesFile(repoFile) + r, err := repo.LoadRepositoryFile(repoFile) if err != nil { return err } diff --git a/cmd/helm/repo_remove_test.go b/cmd/helm/repo_remove_test.go index 12f534972..1e0a4c382 100644 --- a/cmd/helm/repo_remove_test.go +++ b/cmd/helm/repo_remove_test.go @@ -69,7 +69,7 @@ func TestRepoRemove(t *testing.T) { t.Errorf("Error cache file was not removed for repository %s", testName) } - f, err := repo.LoadRepositoriesFile(hh.RepositoryFile()) + f, err := repo.LoadRepositoryFile(hh.RepositoryFile()) if err != nil { t.Error(err) } diff --git a/cmd/helm/repo_update.go b/cmd/helm/repo_update.go index 8dfab9a5d..c08d840fb 100644 --- a/cmd/helm/repo_update.go +++ b/cmd/helm/repo_update.go @@ -65,7 +65,7 @@ func newRepoUpdateCmd(out io.Writer) *cobra.Command { } func (u *repoUpdateCmd) run() error { - f, err := repo.LoadRepositoriesFile(u.home.RepositoryFile()) + f, err := repo.LoadRepositoryFile(u.home.RepositoryFile()) if err != nil { return err } diff --git a/cmd/helm/search.go b/cmd/helm/search.go index 62d13e3b3..4804f6b92 100644 --- a/cmd/helm/search.go +++ b/cmd/helm/search.go @@ -110,7 +110,7 @@ func (s *searchCmd) formatSearchResults(res []*search.Result) string { func (s *searchCmd) buildIndex() (*search.Index, error) { // Load the repositories.yaml - rf, err := repo.LoadRepositoriesFile(s.helmhome.RepositoryFile()) + rf, err := repo.LoadRepositoryFile(s.helmhome.RepositoryFile()) if err != nil { return nil, err } diff --git a/pkg/repo/chartrepo.go b/pkg/repo/chartrepo.go index c83592678..523e5e1e4 100644 --- a/pkg/repo/chartrepo.go +++ b/pkg/repo/chartrepo.go @@ -29,6 +29,7 @@ import ( "k8s.io/helm/pkg/chartutil" "k8s.io/helm/pkg/provenance" "k8s.io/helm/pkg/tlsutil" + "k8s.io/helm/pkg/urlutil" ) // ChartRepositoryConfig represents a collection of parameters for chart repository @@ -58,6 +59,13 @@ func NewChartRepository(cfg *ChartRepositoryConfig) (*ChartRepository, error) { return nil, fmt.Errorf("can't create TLS config for client: %s", err.Error()) } tlsConf.BuildNameToCertificate() + + sni, err := urlutil.ExtractHostname(cfg.URL) + if err != nil { + return nil, err + } + tlsConf.ServerName = sni + client = &http.Client{ Transport: &http.Transport{ TLSClientConfig: tlsConf, diff --git a/pkg/repo/repo.go b/pkg/repo/repo.go index 7d4a91387..bfc7d4c1d 100644 --- a/pkg/repo/repo.go +++ b/pkg/repo/repo.go @@ -48,11 +48,11 @@ func NewRepositoryFile() *RepositoryFile { } } -// LoadRepositoriesFile takes a file at the given path and returns a RepositoryFile object +// LoadRepositoryFile takes a file at the given path and returns a RepositoryFile object // // If this returns ErrRepoOutOfDate, it also returns a recovered RepositoryFile that // can be saved as a replacement to the out of date file. -func LoadRepositoriesFile(path string) (*RepositoryFile, error) { +func LoadRepositoryFile(path string) (*RepositoryFile, error) { b, err := ioutil.ReadFile(path) if err != nil { return nil, err diff --git a/pkg/repo/repo_test.go b/pkg/repo/repo_test.go index 6ee733fcc..451e035aa 100644 --- a/pkg/repo/repo_test.go +++ b/pkg/repo/repo_test.go @@ -73,7 +73,7 @@ func TestNewRepositoriesFile(t *testing.T) { }, ) - repofile, err := LoadRepositoriesFile(testRepositoriesFile) + repofile, err := LoadRepositoryFile(testRepositoriesFile) if err != nil { t.Errorf("%q could not be loaded: %s", testRepositoriesFile, err) } @@ -97,7 +97,7 @@ func TestNewRepositoriesFile(t *testing.T) { } func TestNewPreV1RepositoriesFile(t *testing.T) { - r, err := LoadRepositoriesFile("testdata/old-repositories.yaml") + r, err := LoadRepositoryFile("testdata/old-repositories.yaml") if err != nil && err != ErrRepoOutOfDate { t.Fatal(err) } diff --git a/pkg/urlutil/urlutil.go b/pkg/urlutil/urlutil.go index 8026fd245..f99294938 100644 --- a/pkg/urlutil/urlutil.go +++ b/pkg/urlutil/urlutil.go @@ -17,6 +17,7 @@ limitations under the License. package urlutil import ( + "net" "net/url" "path" "path/filepath" @@ -64,3 +65,17 @@ func URLAreEqual(a, b string) bool { } return au.String() == bu.String() } + +// ExtractHostname returns hostname from URL +func ExtractHostname(addr string) (string, error) { + u, err := url.Parse(addr) + if err != nil { + return "", err + } + + host, _, err := net.SplitHostPort(u.Host) + if err != nil { + return "", err + } + return host, nil +} From a82c19b91ac399ba824e2f386c73d8773396243f Mon Sep 17 00:00:00 2001 From: Anton Galitsyn Date: Wed, 28 Dec 2016 23:20:20 +0700 Subject: [PATCH 3/7] fix rebase problems --- cmd/helm/init_test.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/cmd/helm/init_test.go b/cmd/helm/init_test.go index d30b985aa..d03e1b93f 100644 --- a/cmd/helm/init_test.go +++ b/cmd/helm/init_test.go @@ -43,22 +43,20 @@ func TestInitCmd(t *testing.T) { defer os.Remove(home) var buf bytes.Buffer - - fake := testclient.Fake{} + fc := fake.NewSimpleClientset() cmd := &initCmd{ out: &buf, home: helmpath.Home(home), - kubeClient: fake.Extensions(), + kubeClient: fc.Extensions(), + namespace: api.NamespaceDefault, } if err := cmd.run(); err != nil { t.Errorf("expected error: %v", err) } - - actions := fake.Actions() - if action, ok := actions[0].(testclient.CreateAction); !ok || action.GetResource() != "deployments" { - t.Errorf("unexpected action: %v, expected create deployment", actions[0]) + action := fc.Actions()[0] + if !action.Matches("create", "deployments") { + t.Errorf("unexpected action: %v, expected create deployment", action) } - expected := "Tiller (the helm server side component) has been installed into your Kubernetes Cluster." if !strings.Contains(buf.String(), expected) { t.Errorf("expected %q, got %q", expected, buf.String()) From e3f39f30bfe4af904be9de76df9c47658b39f62b Mon Sep 17 00:00:00 2001 From: Anton Galitsyn Date: Mon, 9 Jan 2017 09:48:51 +0700 Subject: [PATCH 4/7] rename URLAreEqual func --- cmd/helm/downloader/manager.go | 6 +++--- pkg/urlutil/urlutil.go | 6 ++---- pkg/urlutil/urlutil_test.go | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cmd/helm/downloader/manager.go b/cmd/helm/downloader/manager.go index daa65ef97..5c9118888 100644 --- a/cmd/helm/downloader/manager.go +++ b/cmd/helm/downloader/manager.go @@ -227,7 +227,7 @@ func (m *Manager) hasAllRepos(deps []*chartutil.Dependency) error { found = true } else { for _, repo := range repos { - if urlutil.URLAreEqual(repo.URL, strings.TrimSuffix(dd.Repository, "/")) { + if urlutil.Equal(repo.URL, strings.TrimSuffix(dd.Repository, "/")) { found = true } } @@ -259,7 +259,7 @@ func (m *Manager) getRepoNames(deps []*chartutil.Dependency) (map[string]string, found := false for _, repo := range repos { - if urlutil.URLAreEqual(repo.URL, dd.Repository) { + if urlutil.Equal(repo.URL, dd.Repository) { found = true reposMap[dd.Name] = repo.Name break @@ -325,7 +325,7 @@ func (m *Manager) parallelRepoUpdate(repos []*repo.ChartRepositoryConfig) error // If it finds a URL that is "relative", it will prepend the repoURL. func findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (string, error) { for _, cr := range repos { - if urlutil.URLAreEqual(repoURL, cr.Config.URL) { + if urlutil.Equal(repoURL, cr.Config.URL) { entry, err := findEntryByName(name, cr) if err != nil { return "", err diff --git a/pkg/urlutil/urlutil.go b/pkg/urlutil/urlutil.go index f99294938..3a6570470 100644 --- a/pkg/urlutil/urlutil.go +++ b/pkg/urlutil/urlutil.go @@ -41,10 +41,8 @@ func URLJoin(baseURL string, paths ...string) (string, error) { return u.String(), nil } -// URLAreEqual normalizes two URLs and then compares for equality. -// -// TODO: This and the urlJoin functions should really be moved to a 'urlutil' package. -func URLAreEqual(a, b string) bool { +// Equal normalizes two URLs and then compares for equality. +func Equal(a, b string) bool { au, err := url.Parse(a) if err != nil { a = filepath.Clean(a) diff --git a/pkg/urlutil/urlutil_test.go b/pkg/urlutil/urlutil_test.go index d4aa1139d..5944df1ae 100644 --- a/pkg/urlutil/urlutil_test.go +++ b/pkg/urlutil/urlutil_test.go @@ -39,7 +39,7 @@ func TestUrlJoin(t *testing.T) { } } -func TestUrlAreEqual(t *testing.T) { +func TestEqual(t *testing.T) { for _, tt := range []struct { a, b string match bool @@ -57,7 +57,7 @@ func TestUrlAreEqual(t *testing.T) { {"/foo", "/foo/", true}, {"/foo/.", "/foo/", true}, } { - if tt.match != URLAreEqual(tt.a, tt.b) { + if tt.match != Equal(tt.a, tt.b) { t.Errorf("Expected %q==%q to be %t", tt.a, tt.b, tt.match) } } From 8a1d43ec3f8d0e0eedb371466cf8394219888e81 Mon Sep 17 00:00:00 2001 From: Anton Galitsyn Date: Mon, 9 Jan 2017 09:49:11 +0700 Subject: [PATCH 5/7] check the error that WriteFile returns --- cmd/helm/init.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/helm/init.go b/cmd/helm/init.go index 43a0ece81..697e1bed7 100644 --- a/cmd/helm/init.go +++ b/cmd/helm/init.go @@ -190,7 +190,9 @@ func ensureDefaultRepos(home helmpath.Home, out io.Writer) error { } f.Add(sr) f.Add(lr) - f.WriteFile(repoFile, 0644) + if err := f.WriteFile(repoFile, 0644); err != nil { + return err + } } else if fi.IsDir() { return fmt.Errorf("%s must be a file, not a directory", repoFile) } From 685e730ba8faf88d444131506a90943fad54e264 Mon Sep 17 00:00:00 2001 From: Anton Galitsyn Date: Mon, 9 Jan 2017 17:53:33 +0700 Subject: [PATCH 6/7] create repo.Getter interface --- cmd/helm/downloader/chart_downloader.go | 17 ++++++++--------- pkg/repo/chartrepo.go | 14 +++++++++++++- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cmd/helm/downloader/chart_downloader.go b/cmd/helm/downloader/chart_downloader.go index 57e79e9c8..610eeca22 100644 --- a/cmd/helm/downloader/chart_downloader.go +++ b/cmd/helm/downloader/chart_downloader.go @@ -21,7 +21,6 @@ import ( "fmt" "io" "io/ioutil" - "net/http" "net/url" "os" "path/filepath" @@ -76,12 +75,12 @@ type ChartDownloader struct { // Returns a string path to the location where the file was downloaded and a verification // (if provenance was verified), or an error if something bad happened. func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *provenance.Verification, error) { - u, client, err := c.ResolveChartVersion(ref, version) + u, r, err := c.ResolveChartVersion(ref, version) if err != nil { return "", nil, err } - data, err := download(u.String(), client) + data, err := download(u.String(), r) if err != nil { return "", nil, err } @@ -95,7 +94,7 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven // If provenance is requested, verify it. ver := &provenance.Verification{} if c.Verify > VerifyNever { - body, err := download(u.String()+".prov", client) + body, err := download(u.String()+".prov", r) if err != nil { if c.Verify == VerifyAlways { return destfile, ver, fmt.Errorf("Failed to fetch provenance %q", u.String()+".prov") @@ -131,7 +130,7 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven // * If version is non-empty, this will return the URL for that version // * If version is empty, this will return the URL for the latest version // * If no version can be found, an error is returned -func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, *http.Client, error) { +func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, *repo.ChartRepository, error) { u, err := url.Parse(ref) if err != nil { return nil, nil, fmt.Errorf("invalid chart URL format: %s", ref) @@ -199,7 +198,7 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, *h return nil, nil, fmt.Errorf("invalid chart URL format: %s", ref) } - return u, r.Client, nil + return u, r, nil } // VerifyChart takes a path to a chart archive and a keyring, and verifies the chart. @@ -228,11 +227,11 @@ func VerifyChart(path string, keyring string) (*provenance.Verification, error) return sig.Verify(path, provfile) } -// download performs a HTTP Get using specified client and returns the body. -func download(href string, client *http.Client) (*bytes.Buffer, error) { +// download performs a Get from repo.Getter and returns the body. +func download(href string, r repo.Getter) (*bytes.Buffer, error) { buf := bytes.NewBuffer(nil) - resp, err := client.Get(href) + resp, err := r.Get(href) if err != nil { return buf, err } diff --git a/pkg/repo/chartrepo.go b/pkg/repo/chartrepo.go index 523e5e1e4..a2991ce79 100644 --- a/pkg/repo/chartrepo.go +++ b/pkg/repo/chartrepo.go @@ -50,6 +50,10 @@ type ChartRepository struct { Client *http.Client } +type Getter interface { + Get(url string) (*http.Response, error) +} + // NewChartRepository constructs ChartRepository func NewChartRepository(cfg *ChartRepositoryConfig) (*ChartRepository, error) { var client *http.Client @@ -82,6 +86,14 @@ func NewChartRepository(cfg *ChartRepositoryConfig) (*ChartRepository, error) { }, nil } +func (r *ChartRepository) Get(url string) (*http.Response, error) { + resp, err := r.Client.Get(url) + if err != nil { + return nil, err + } + return resp, nil +} + // Load loads a directory of charts as if it were a repository. // // It requires the presence of an index.yaml file in the directory. @@ -119,7 +131,7 @@ func (r *ChartRepository) DownloadIndexFile() error { var indexURL string indexURL = strings.TrimSuffix(r.Config.URL, "/") + "/index.yaml" - resp, err := r.Client.Get(indexURL) + resp, err := r.Get(indexURL) if err != nil { return err } From f8a261f277d29d5ccff177b5fe6cf3781301fd70 Mon Sep 17 00:00:00 2001 From: Anton Galitsyn Date: Mon, 9 Jan 2017 18:25:37 +0700 Subject: [PATCH 7/7] use old naming --- cmd/helm/dependency_build_test.go | 2 +- cmd/helm/dependency_update_test.go | 2 +- cmd/helm/downloader/chart_downloader.go | 10 +- cmd/helm/downloader/manager.go | 12 +- cmd/helm/helm_test.go | 10 +- cmd/helm/init.go | 14 +- cmd/helm/repo_add.go | 4 +- cmd/helm/repo_add_test.go | 2 +- cmd/helm/repo_index.go | 4 +- cmd/helm/repo_index_test.go | 4 +- cmd/helm/repo_list.go | 2 +- cmd/helm/repo_remove.go | 2 +- cmd/helm/repo_remove_test.go | 2 +- cmd/helm/repo_update.go | 2 +- cmd/helm/repo_update_test.go | 2 +- cmd/helm/resolver/resolver.go | 2 +- cmd/helm/search.go | 4 +- cmd/helm/search/search.go | 2 +- cmd/helm/search/search_test.go | 4 +- pkg/repo/chartrepo.go | 16 +- pkg/repo/chartrepo_test.go | 10 +- pkg/repo/index.go | 224 ++++++++++++------------ pkg/repo/index_test.go | 16 +- pkg/repo/local.go | 4 +- pkg/repo/repo.go | 42 ++--- pkg/repo/repo_test.go | 16 +- pkg/repo/repotest/server.go | 6 +- pkg/repo/repotest/server_test.go | 2 +- 28 files changed, 213 insertions(+), 209 deletions(-) diff --git a/cmd/helm/dependency_build_test.go b/cmd/helm/dependency_build_test.go index 9e615a994..361d3ed6c 100644 --- a/cmd/helm/dependency_build_test.go +++ b/cmd/helm/dependency_build_test.go @@ -103,7 +103,7 @@ func TestDependencyBuildCmd(t *testing.T) { t.Fatal(err) } - i, err := repo.NewChartRepositoryIndexFromFile(dbc.helmhome.CacheIndex("test")) + i, err := repo.LoadIndexFile(dbc.helmhome.CacheIndex("test")) if err != nil { t.Fatal(err) } diff --git a/cmd/helm/dependency_update_test.go b/cmd/helm/dependency_update_test.go index 9c6171fe7..e47e494f5 100644 --- a/cmd/helm/dependency_update_test.go +++ b/cmd/helm/dependency_update_test.go @@ -88,7 +88,7 @@ func TestDependencyUpdateCmd(t *testing.T) { t.Fatal(err) } - i, err := repo.NewChartRepositoryIndexFromFile(duc.helmhome.CacheIndex("test")) + i, err := repo.LoadIndexFile(duc.helmhome.CacheIndex("test")) if err != nil { t.Fatal(err) } diff --git a/cmd/helm/downloader/chart_downloader.go b/cmd/helm/downloader/chart_downloader.go index 610eeca22..2b239d03d 100644 --- a/cmd/helm/downloader/chart_downloader.go +++ b/cmd/helm/downloader/chart_downloader.go @@ -136,14 +136,14 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, *r return nil, nil, fmt.Errorf("invalid chart URL format: %s", ref) } - rf, err := repo.LoadRepositoryFile(c.HelmHome.RepositoryFile()) + rf, err := repo.LoadRepositoriesFile(c.HelmHome.RepositoryFile()) if err != nil { return nil, nil, err } var ( chartName string - rc *repo.ChartRepositoryConfig + rc *repo.Entry ) if u.IsAbs() && len(u.Host) > 0 && len(u.Path) > 0 { // If it has a scheme and host and path, it's a full URL @@ -178,7 +178,7 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, *r } // Next, we need to load the index, and actually look up the chart. - i, err := repo.NewChartRepositoryIndexFromFile(c.HelmHome.CacheIndex(r.Config.Name)) + i, err := repo.LoadIndexFile(c.HelmHome.CacheIndex(r.Config.Name)) if err != nil { return nil, nil, fmt.Errorf("no cached repo found. (try 'helm repo update'). %s", err) } @@ -252,7 +252,7 @@ func isTar(filename string) bool { return strings.ToLower(filepath.Ext(filename)) == ".tgz" } -func pickChartRepositoryConfigByName(name string, cfgs []*repo.ChartRepositoryConfig) (*repo.ChartRepositoryConfig, error) { +func pickChartRepositoryConfigByName(name string, cfgs []*repo.Entry) (*repo.Entry, error) { for _, rc := range cfgs { if rc.Name == name { if rc.URL == "" { @@ -264,7 +264,7 @@ func pickChartRepositoryConfigByName(name string, cfgs []*repo.ChartRepositoryCo return nil, fmt.Errorf("repo %s not found", name) } -func pickChartRepositoryConfigByURL(u string, cfgs []*repo.ChartRepositoryConfig) (*repo.ChartRepositoryConfig, error) { +func pickChartRepositoryConfigByURL(u string, cfgs []*repo.Entry) (*repo.Entry, error) { for _, rc := range cfgs { if rc.URL == u { return rc, nil diff --git a/cmd/helm/downloader/manager.go b/cmd/helm/downloader/manager.go index 5c9118888..a29dad99b 100644 --- a/cmd/helm/downloader/manager.go +++ b/cmd/helm/downloader/manager.go @@ -213,7 +213,7 @@ func (m *Manager) downloadAll(deps []*chartutil.Dependency) error { // hasAllRepos ensures that all of the referenced deps are in the local repo cache. func (m *Manager) hasAllRepos(deps []*chartutil.Dependency) error { - rf, err := repo.LoadRepositoryFile(m.HelmHome.RepositoryFile()) + rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) if err != nil { return err } @@ -244,7 +244,7 @@ func (m *Manager) hasAllRepos(deps []*chartutil.Dependency) error { // getRepoNames returns the repo names of the referenced deps which can be used to fetch the cahced index file. func (m *Manager) getRepoNames(deps []*chartutil.Dependency) (map[string]string, error) { - rf, err := repo.LoadRepositoryFile(m.HelmHome.RepositoryFile()) + rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) if err != nil { return nil, err } @@ -277,7 +277,7 @@ func (m *Manager) getRepoNames(deps []*chartutil.Dependency) (map[string]string, // UpdateRepositories updates all of the local repos to the latest. func (m *Manager) UpdateRepositories() error { - rf, err := repo.LoadRepositoryFile(m.HelmHome.RepositoryFile()) + rf, err := repo.LoadRepositoriesFile(m.HelmHome.RepositoryFile()) if err != nil { return err } @@ -291,7 +291,7 @@ func (m *Manager) UpdateRepositories() error { return nil } -func (m *Manager) parallelRepoUpdate(repos []*repo.ChartRepositoryConfig) error { +func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error { out := m.Out fmt.Fprintln(out, "Hang tight while we grab the latest from your chart repositories...") var wg sync.WaitGroup @@ -409,7 +409,7 @@ func (m *Manager) loadChartRepositories() (map[string]*repo.ChartRepository, err repoyaml := m.HelmHome.RepositoryFile() // Load repositories.yaml file - rf, err := repo.LoadRepositoryFile(repoyaml) + rf, err := repo.LoadRepositoriesFile(repoyaml) if err != nil { return indices, fmt.Errorf("failed to load %s: %s", repoyaml, err) } @@ -417,7 +417,7 @@ func (m *Manager) loadChartRepositories() (map[string]*repo.ChartRepository, err for _, re := range rf.Repositories { lname := re.Name cacheindex := m.HelmHome.CacheIndex(lname) - index, err := repo.NewChartRepositoryIndexFromFile(cacheindex) + index, err := repo.LoadIndexFile(cacheindex) if err != nil { return indices, err } diff --git a/cmd/helm/helm_test.go b/cmd/helm/helm_test.go index 7dfc34dae..682709f2d 100644 --- a/cmd/helm/helm_test.go +++ b/cmd/helm/helm_test.go @@ -254,12 +254,12 @@ func ensureTestHome(home helmpath.Home, t *testing.T) error { repoFile := home.RepositoryFile() if fi, err := os.Stat(repoFile); err != nil { - rf := repo.NewRepositoryFile() - rf.Add(&repo.ChartRepositoryConfig{ + rf := repo.NewRepoFile() + rf.Add(&repo.Entry{ Name: "charts", URL: "http://example.com/foo", Cache: "charts-index.yaml", - }, &repo.ChartRepositoryConfig{ + }, &repo.Entry{ Name: "local", URL: "http://localhost.com:7743/foo", Cache: "local-index.yaml", @@ -270,7 +270,7 @@ func ensureTestHome(home helmpath.Home, t *testing.T) error { } else if fi.IsDir() { return fmt.Errorf("%s must be a file, not a directory", repoFile) } - if r, err := repo.LoadRepositoryFile(repoFile); err == repo.ErrRepoOutOfDate { + if r, err := repo.LoadRepositoriesFile(repoFile); err == repo.ErrRepoOutOfDate { t.Log("Updating repository file format...") if err := r.WriteFile(repoFile, 0644); err != nil { return err @@ -279,7 +279,7 @@ func ensureTestHome(home helmpath.Home, t *testing.T) error { localRepoIndexFile := home.LocalRepository(localRepoIndexFilePath) if fi, err := os.Stat(localRepoIndexFile); err != nil { - i := repo.NewChartRepositoryIndex() + i := repo.NewIndexFile() if err := i.WriteFile(localRepoIndexFile, 0644); err != nil { return err } diff --git a/cmd/helm/init.go b/cmd/helm/init.go index 697e1bed7..b9e807cb0 100644 --- a/cmd/helm/init.go +++ b/cmd/helm/init.go @@ -179,7 +179,7 @@ func ensureDefaultRepos(home helmpath.Home, out io.Writer) error { repoFile := home.RepositoryFile() if fi, err := os.Stat(repoFile); err != nil { fmt.Fprintf(out, "Creating %s \n", repoFile) - f := repo.NewRepositoryFile() + f := repo.NewRepoFile() sr, err := initStableRepo(home.CacheIndex(stableRepository)) if err != nil { return err @@ -199,8 +199,8 @@ func ensureDefaultRepos(home helmpath.Home, out io.Writer) error { return nil } -func initStableRepo(cacheFile string) (*repo.ChartRepositoryConfig, error) { - c := repo.ChartRepositoryConfig{ +func initStableRepo(cacheFile string) (*repo.Entry, error) { + c := repo.Entry{ Name: stableRepository, URL: stableRepositoryURL, Cache: cacheFile, @@ -217,9 +217,9 @@ func initStableRepo(cacheFile string) (*repo.ChartRepositoryConfig, error) { return &c, nil } -func initLocalRepo(indexFile, cacheFile string) (*repo.ChartRepositoryConfig, error) { +func initLocalRepo(indexFile, cacheFile string) (*repo.Entry, error) { if fi, err := os.Stat(indexFile); err != nil { - i := repo.NewChartRepositoryIndex() + i := repo.NewIndexFile() if err := i.WriteFile(indexFile, 0644); err != nil { return nil, err } @@ -230,7 +230,7 @@ func initLocalRepo(indexFile, cacheFile string) (*repo.ChartRepositoryConfig, er return nil, fmt.Errorf("%s must be a file, not a directory", indexFile) } - return &repo.ChartRepositoryConfig{ + return &repo.Entry{ Name: localRepository, URL: localRepositoryURL, Cache: cacheFile, @@ -238,7 +238,7 @@ func initLocalRepo(indexFile, cacheFile string) (*repo.ChartRepositoryConfig, er } func ensureRepoFileFormat(file string, out io.Writer) error { - r, err := repo.LoadRepositoryFile(file) + r, err := repo.LoadRepositoriesFile(file) if err == repo.ErrRepoOutOfDate { fmt.Fprintln(out, "Updating repository file format...") if err := r.WriteFile(file, 0644); err != nil { diff --git a/cmd/helm/repo_add.go b/cmd/helm/repo_add.go index 2d9a62d1e..b349537a7 100644 --- a/cmd/helm/repo_add.go +++ b/cmd/helm/repo_add.go @@ -78,7 +78,7 @@ func (a *repoAddCmd) run() error { } func addRepository(name, url string, home helmpath.Home, certFile, keyFile, caFile string, noUpdate bool) error { - f, err := repo.LoadRepositoryFile(home.RepositoryFile()) + f, err := repo.LoadRepositoriesFile(home.RepositoryFile()) if err != nil { return err } @@ -88,7 +88,7 @@ func addRepository(name, url string, home helmpath.Home, certFile, keyFile, caFi } cif := home.CacheIndex(name) - c := repo.ChartRepositoryConfig{ + c := repo.Entry{ Name: name, Cache: cif, URL: url, diff --git a/cmd/helm/repo_add_test.go b/cmd/helm/repo_add_test.go index b823777f7..7c5342177 100644 --- a/cmd/helm/repo_add_test.go +++ b/cmd/helm/repo_add_test.go @@ -84,7 +84,7 @@ func TestRepoAdd(t *testing.T) { t.Error(err) } - f, err := repo.LoadRepositoryFile(hh.RepositoryFile()) + f, err := repo.LoadRepositoriesFile(hh.RepositoryFile()) if err != nil { t.Error(err) } diff --git a/cmd/helm/repo_index.go b/cmd/helm/repo_index.go index 925dffc17..ad5808946 100644 --- a/cmd/helm/repo_index.go +++ b/cmd/helm/repo_index.go @@ -83,12 +83,12 @@ func (i *repoIndexCmd) run() error { func index(dir, url, mergeTo string) error { out := filepath.Join(dir, "index.yaml") - i, err := repo.NewChartRepositoryIndexFromDirectory(dir, url) + i, err := repo.IndexDirectory(dir, url) if err != nil { return err } if mergeTo != "" { - i2, err := repo.NewChartRepositoryIndexFromFile(mergeTo) + i2, err := repo.LoadIndexFile(mergeTo) if err != nil { return fmt.Errorf("Merge failed: %s", err) } diff --git a/cmd/helm/repo_index_test.go b/cmd/helm/repo_index_test.go index b3e37e4b1..bd1010dd7 100644 --- a/cmd/helm/repo_index_test.go +++ b/cmd/helm/repo_index_test.go @@ -53,7 +53,7 @@ func TestRepoIndexCmd(t *testing.T) { destIndex := filepath.Join(dir, "index.yaml") - index, err := repo.NewChartRepositoryIndexFromFile(destIndex) + index, err := repo.LoadIndexFile(destIndex) if err != nil { t.Fatal(err) } @@ -94,7 +94,7 @@ func TestRepoIndexCmd(t *testing.T) { t.Error(err) } - index, err = repo.NewChartRepositoryIndexFromFile(destIndex) + index, err = repo.LoadIndexFile(destIndex) if err != nil { t.Fatal(err) } diff --git a/cmd/helm/repo_list.go b/cmd/helm/repo_list.go index a4f7edec5..a3816facd 100644 --- a/cmd/helm/repo_list.go +++ b/cmd/helm/repo_list.go @@ -51,7 +51,7 @@ func newRepoListCmd(out io.Writer) *cobra.Command { } func (a *repoListCmd) run() error { - f, err := repo.LoadRepositoryFile(a.home.RepositoryFile()) + f, err := repo.LoadRepositoriesFile(a.home.RepositoryFile()) if err != nil { return err } diff --git a/cmd/helm/repo_remove.go b/cmd/helm/repo_remove.go index ad2a9c188..f7671cab0 100644 --- a/cmd/helm/repo_remove.go +++ b/cmd/helm/repo_remove.go @@ -62,7 +62,7 @@ func (r *repoRemoveCmd) run() error { func removeRepoLine(out io.Writer, name string, home helmpath.Home) error { repoFile := home.RepositoryFile() - r, err := repo.LoadRepositoryFile(repoFile) + r, err := repo.LoadRepositoriesFile(repoFile) if err != nil { return err } diff --git a/cmd/helm/repo_remove_test.go b/cmd/helm/repo_remove_test.go index 1e0a4c382..12f534972 100644 --- a/cmd/helm/repo_remove_test.go +++ b/cmd/helm/repo_remove_test.go @@ -69,7 +69,7 @@ func TestRepoRemove(t *testing.T) { t.Errorf("Error cache file was not removed for repository %s", testName) } - f, err := repo.LoadRepositoryFile(hh.RepositoryFile()) + f, err := repo.LoadRepositoriesFile(hh.RepositoryFile()) if err != nil { t.Error(err) } diff --git a/cmd/helm/repo_update.go b/cmd/helm/repo_update.go index c08d840fb..8dfab9a5d 100644 --- a/cmd/helm/repo_update.go +++ b/cmd/helm/repo_update.go @@ -65,7 +65,7 @@ func newRepoUpdateCmd(out io.Writer) *cobra.Command { } func (u *repoUpdateCmd) run() error { - f, err := repo.LoadRepositoryFile(u.home.RepositoryFile()) + f, err := repo.LoadRepositoriesFile(u.home.RepositoryFile()) if err != nil { return err } diff --git a/cmd/helm/repo_update_test.go b/cmd/helm/repo_update_test.go index f7be052b6..97f023f13 100644 --- a/cmd/helm/repo_update_test.go +++ b/cmd/helm/repo_update_test.go @@ -80,7 +80,7 @@ func TestUpdateCharts(t *testing.T) { t.Fatal(err) } - r, err := repo.NewChartRepository(&repo.ChartRepositoryConfig{ + r, err := repo.NewChartRepository(&repo.Entry{ Name: "charts", URL: ts.URL(), Cache: hh.CacheIndex("charts"), diff --git a/cmd/helm/resolver/resolver.go b/cmd/helm/resolver/resolver.go index dfa1e74a7..480cea674 100644 --- a/cmd/helm/resolver/resolver.go +++ b/cmd/helm/resolver/resolver.go @@ -60,7 +60,7 @@ func (r *Resolver) Resolve(reqs *chartutil.Requirements, repoNames map[string]st return nil, fmt.Errorf("dependency %q has an invalid version/constraint format: %s", d.Name, err) } - repoIndex, err := repo.NewChartRepositoryIndexFromFile(r.helmhome.CacheIndex(repoNames[d.Name])) + repoIndex, err := repo.LoadIndexFile(r.helmhome.CacheIndex(repoNames[d.Name])) if err != nil { return nil, fmt.Errorf("no cached repo found. (try 'helm repo update'). %s", err) } diff --git a/cmd/helm/search.go b/cmd/helm/search.go index 4804f6b92..a1d57adb0 100644 --- a/cmd/helm/search.go +++ b/cmd/helm/search.go @@ -110,7 +110,7 @@ func (s *searchCmd) formatSearchResults(res []*search.Result) string { func (s *searchCmd) buildIndex() (*search.Index, error) { // Load the repositories.yaml - rf, err := repo.LoadRepositoryFile(s.helmhome.RepositoryFile()) + rf, err := repo.LoadRepositoriesFile(s.helmhome.RepositoryFile()) if err != nil { return nil, err } @@ -119,7 +119,7 @@ func (s *searchCmd) buildIndex() (*search.Index, error) { for _, re := range rf.Repositories { n := re.Name f := s.helmhome.CacheIndex(n) - ind, err := repo.NewChartRepositoryIndexFromFile(f) + ind, err := repo.LoadIndexFile(f) if err != nil { fmt.Fprintf(s.out, "WARNING: Repo %q is corrupt or missing. Try 'helm repo update'.", n) continue diff --git a/cmd/helm/search/search.go b/cmd/helm/search/search.go index 5675332df..4d394e0f5 100644 --- a/cmd/helm/search/search.go +++ b/cmd/helm/search/search.go @@ -61,7 +61,7 @@ func NewIndex() *Index { const verSep = "$$" // AddRepo adds a repository index to the search index. -func (i *Index) AddRepo(rname string, ind *repo.ChartRepositoryIndex, all bool) { +func (i *Index) AddRepo(rname string, ind *repo.IndexFile, all bool) { for name, ref := range ind.Entries { if len(ref) == 0 { // Skip chart names that have zero releases. diff --git a/cmd/helm/search/search_test.go b/cmd/helm/search/search_test.go index 1a607fdc7..7f5d29409 100644 --- a/cmd/helm/search/search_test.go +++ b/cmd/helm/search/search_test.go @@ -95,8 +95,8 @@ var indexfileEntries = map[string]repo.ChartVersions{ func loadTestIndex(t *testing.T, all bool) *Index { i := NewIndex() - i.AddRepo("testing", &repo.ChartRepositoryIndex{Entries: indexfileEntries}, all) - i.AddRepo("ztesting", &repo.ChartRepositoryIndex{Entries: map[string]repo.ChartVersions{ + i.AddRepo("testing", &repo.IndexFile{Entries: indexfileEntries}, all) + i.AddRepo("ztesting", &repo.IndexFile{Entries: map[string]repo.ChartVersions{ "pinta": { { URLs: []string{"http://example.com/charts/pinta-2.0.0.tgz"}, diff --git a/pkg/repo/chartrepo.go b/pkg/repo/chartrepo.go index a2991ce79..6e2a3676c 100644 --- a/pkg/repo/chartrepo.go +++ b/pkg/repo/chartrepo.go @@ -32,8 +32,8 @@ import ( "k8s.io/helm/pkg/urlutil" ) -// ChartRepositoryConfig represents a collection of parameters for chart repository -type ChartRepositoryConfig struct { +// Entry represents a collection of parameters for chart repository +type Entry struct { Name string `json:"name"` Cache string `json:"cache"` URL string `json:"url"` @@ -44,18 +44,19 @@ type ChartRepositoryConfig struct { // ChartRepository represents a chart repository type ChartRepository struct { - Config *ChartRepositoryConfig + Config *Entry ChartPaths []string - IndexFile *ChartRepositoryIndex + IndexFile *IndexFile Client *http.Client } +// Getter is an interface to support GET to the specified URL. type Getter interface { Get(url string) (*http.Response, error) } // NewChartRepository constructs ChartRepository -func NewChartRepository(cfg *ChartRepositoryConfig) (*ChartRepository, error) { +func NewChartRepository(cfg *Entry) (*ChartRepository, error) { var client *http.Client if cfg.CertFile != "" && cfg.KeyFile != "" && cfg.CAFile != "" { tlsConf, err := tlsutil.NewClientTLS(cfg.CertFile, cfg.KeyFile, cfg.CAFile) @@ -81,11 +82,12 @@ func NewChartRepository(cfg *ChartRepositoryConfig) (*ChartRepository, error) { return &ChartRepository{ Config: cfg, - IndexFile: NewChartRepositoryIndex(), + IndexFile: NewIndexFile(), Client: client, }, nil } +// Get issues a GET using configured client to the specified URL. func (r *ChartRepository) Get(url string) (*http.Response, error) { resp, err := r.Client.Get(url) if err != nil { @@ -112,7 +114,7 @@ func (r *ChartRepository) Load() error { filepath.Walk(r.Config.Name, func(path string, f os.FileInfo, err error) error { if !f.IsDir() { if strings.Contains(f.Name(), "-index.yaml") { - i, err := NewChartRepositoryIndexFromFile(path) + i, err := LoadIndexFile(path) if err != nil { return nil } diff --git a/pkg/repo/chartrepo_test.go b/pkg/repo/chartrepo_test.go index b9a83a4c2..7148e39a4 100644 --- a/pkg/repo/chartrepo_test.go +++ b/pkg/repo/chartrepo_test.go @@ -32,7 +32,7 @@ const ( ) func TestLoadChartRepository(t *testing.T) { - r, err := NewChartRepository(&ChartRepositoryConfig{ + r, err := NewChartRepository(&Entry{ Name: testRepository, URL: testURL, }) @@ -64,7 +64,7 @@ func TestLoadChartRepository(t *testing.T) { } func TestIndex(t *testing.T) { - r, err := NewChartRepository(&ChartRepositoryConfig{ + r, err := NewChartRepository(&Entry{ Name: testRepository, URL: testURL, }) @@ -82,7 +82,7 @@ func TestIndex(t *testing.T) { } tempIndexPath := filepath.Join(testRepository, indexPath) - actual, err := NewChartRepositoryIndexFromFile(tempIndexPath) + actual, err := LoadIndexFile(tempIndexPath) defer os.Remove(tempIndexPath) // clean up if err != nil { t.Errorf("Error loading index file %v", err) @@ -94,14 +94,14 @@ func TestIndex(t *testing.T) { if err != nil { t.Errorf("Error performing re-index: %s\n", err) } - second, err := NewChartRepositoryIndexFromFile(tempIndexPath) + second, err := LoadIndexFile(tempIndexPath) if err != nil { t.Errorf("Error re-loading index file %v", err) } verifyIndex(t, second) } -func verifyIndex(t *testing.T, actual *ChartRepositoryIndex) { +func verifyIndex(t *testing.T, actual *IndexFile) { var empty time.Time if actual.Generated == empty { t.Errorf("Generated should be greater than 0: %s", actual.Generated) diff --git a/pkg/repo/index.go b/pkg/repo/index.go index 1676b118e..148caac4a 100644 --- a/pkg/repo/index.go +++ b/pkg/repo/index.go @@ -74,35 +74,17 @@ func (c ChartVersions) Less(a, b int) bool { return i.LessThan(j) } -// ChartRepositoryIndex represents the index file in a chart repository -type ChartRepositoryIndex struct { +// IndexFile represents the index file in a chart repository +type IndexFile struct { APIVersion string `json:"apiVersion"` Generated time.Time `json:"generated"` Entries map[string]ChartVersions `json:"entries"` PublicKeys []string `json:"publicKeys,omitempty"` } -// ChartVersion represents a chart entry in the ChartRepositoryIndex -type ChartVersion struct { - *chart.Metadata - URLs []string `json:"urls"` - Created time.Time `json:"created,omitempty"` - Removed bool `json:"removed,omitempty"` - Digest string `json:"digest,omitempty"` -} - -// unversionedEntry represents a deprecated pre-Alpha.5 format. -// -// This will be removed prior to v2.0.0 -type unversionedEntry struct { - Checksum string `json:"checksum"` - URL string `json:"url"` - Chartfile *chart.Metadata `json:"chartfile"` -} - -// NewChartRepositoryIndex initializes an index. -func NewChartRepositoryIndex() *ChartRepositoryIndex { - return &ChartRepositoryIndex{ +// NewIndexFile initializes an index. +func NewIndexFile() *IndexFile { + return &IndexFile{ APIVersion: APIVersionV1, Generated: time.Now(), Entries: map[string]ChartVersions{}, @@ -110,8 +92,8 @@ func NewChartRepositoryIndex() *ChartRepositoryIndex { } } -// NewChartRepositoryIndexFromFile takes a file at the given path and returns an ChartRepositoryIndex object -func NewChartRepositoryIndexFromFile(path string) (*ChartRepositoryIndex, error) { +// LoadIndexFile takes a file at the given path and returns an IndexFile object +func LoadIndexFile(path string) (*IndexFile, error) { b, err := ioutil.ReadFile(path) if err != nil { return nil, err @@ -119,89 +101,9 @@ func NewChartRepositoryIndexFromFile(path string) (*ChartRepositoryIndex, error) return loadIndex(b) } -// NewChartRepositoryIndexFromDirectory reads a (flat) directory and generates an index. -// -// It indexes only charts that have been packaged (*.tgz). -// -// The index returned will be in an unsorted state -func NewChartRepositoryIndexFromDirectory(dir, baseURL string) (*ChartRepositoryIndex, error) { - archives, err := filepath.Glob(filepath.Join(dir, "*.tgz")) - if err != nil { - return nil, err - } - index := NewChartRepositoryIndex() - for _, arch := range archives { - fname := filepath.Base(arch) - c, err := chartutil.Load(arch) - if err != nil { - // Assume this is not a chart. - continue - } - hash, err := provenance.DigestFile(arch) - if err != nil { - return index, err - } - index.Add(c.Metadata, fname, baseURL, hash) - } - return index, nil -} - -// loadIndex loads an index file and does minimal validity checking. -// -// This will fail if API Version is not set (ErrNoAPIVersion) or if the unmarshal fails. -func loadIndex(data []byte) (*ChartRepositoryIndex, error) { - i := &ChartRepositoryIndex{} - if err := yaml.Unmarshal(data, i); err != nil { - return i, err - } - if i.APIVersion == "" { - // When we leave Beta, we should remove legacy support and just - // return this error: - //return i, ErrNoAPIVersion - return loadUnversionedIndex(data) - } - return i, nil -} - -// loadUnversionedIndex loads a pre-Alpha.5 index.yaml file. -// -// This format is deprecated. This function will be removed prior to v2.0.0. -func loadUnversionedIndex(data []byte) (*ChartRepositoryIndex, error) { - fmt.Fprintln(os.Stderr, "WARNING: Deprecated index file format. Try 'helm repo update'") - i := map[string]unversionedEntry{} - - // This gets around an error in the YAML parser. Instead of parsing as YAML, - // we convert to JSON, and then decode again. - var err error - data, err = yaml.YAMLToJSON(data) - if err != nil { - return nil, err - } - if err := json.Unmarshal(data, &i); err != nil { - return nil, err - } - - if len(i) == 0 { - return nil, ErrNoAPIVersion - } - ni := NewChartRepositoryIndex() - for n, item := range i { - if item.Chartfile == nil || item.Chartfile.Name == "" { - parts := strings.Split(n, "-") - ver := "" - if len(parts) > 1 { - ver = strings.TrimSuffix(parts[1], ".tgz") - } - item.Chartfile = &chart.Metadata{Name: parts[0], Version: ver} - } - ni.Add(item.Chartfile, item.URL, "", item.Checksum) - } - return ni, nil -} - // Add adds a file to the index // This can leave the index in an unsorted state -func (i ChartRepositoryIndex) Add(md *chart.Metadata, filename, baseURL, digest string) { +func (i IndexFile) Add(md *chart.Metadata, filename, baseURL, digest string) { u := filename if baseURL != "" { var err error @@ -225,7 +127,7 @@ func (i ChartRepositoryIndex) Add(md *chart.Metadata, filename, baseURL, digest } // Has returns true if the index has an entry for a chart with the given name and exact version. -func (i ChartRepositoryIndex) Has(name, version string) bool { +func (i IndexFile) Has(name, version string) bool { _, err := i.Get(name, version) return err == nil } @@ -236,7 +138,7 @@ func (i ChartRepositoryIndex) Has(name, version string) bool { // the most recent release for every version is in the 0th slot in the // Entries.ChartVersions array. That way, tooling can predict the newest // version without needing to parse SemVers. -func (i ChartRepositoryIndex) SortEntries() { +func (i IndexFile) SortEntries() { for _, versions := range i.Entries { sort.Sort(sort.Reverse(versions)) } @@ -245,7 +147,7 @@ func (i ChartRepositoryIndex) SortEntries() { // Get returns the ChartVersion for the given name. // // If version is empty, this will return the chart with the highest version. -func (i ChartRepositoryIndex) Get(name, version string) (*ChartVersion, error) { +func (i IndexFile) Get(name, version string) (*ChartVersion, error) { vs, ok := i.Entries[name] if !ok { return nil, ErrNoChartName @@ -268,7 +170,7 @@ func (i ChartRepositoryIndex) Get(name, version string) (*ChartVersion, error) { // WriteFile writes an index file to the given destination path. // // The mode on the file is set to 'mode'. -func (i ChartRepositoryIndex) WriteFile(dest string, mode os.FileMode) error { +func (i IndexFile) WriteFile(dest string, mode os.FileMode) error { b, err := yaml.Marshal(i) if err != nil { return err @@ -284,7 +186,7 @@ func (i ChartRepositoryIndex) WriteFile(dest string, mode os.FileMode) error { // In all other cases, the existing record is preserved. // // This can leave the index in an unsorted state -func (i *ChartRepositoryIndex) Merge(f *ChartRepositoryIndex) { +func (i *IndexFile) Merge(f *IndexFile) { for _, cvs := range f.Entries { for _, cv := range cvs { if !i.Has(cv.Name, cv.Version) { @@ -294,3 +196,103 @@ func (i *ChartRepositoryIndex) Merge(f *ChartRepositoryIndex) { } } } + +// Need both JSON and YAML annotations until we get rid of gopkg.in/yaml.v2 + +// ChartVersion represents a chart entry in the IndexFile +type ChartVersion struct { + *chart.Metadata + URLs []string `json:"urls"` + Created time.Time `json:"created,omitempty"` + Removed bool `json:"removed,omitempty"` + Digest string `json:"digest,omitempty"` +} + +// IndexDirectory reads a (flat) directory and generates an index. +// +// It indexes only charts that have been packaged (*.tgz). +// +// The index returned will be in an unsorted state +func IndexDirectory(dir, baseURL string) (*IndexFile, error) { + archives, err := filepath.Glob(filepath.Join(dir, "*.tgz")) + if err != nil { + return nil, err + } + index := NewIndexFile() + for _, arch := range archives { + fname := filepath.Base(arch) + c, err := chartutil.Load(arch) + if err != nil { + // Assume this is not a chart. + continue + } + hash, err := provenance.DigestFile(arch) + if err != nil { + return index, err + } + index.Add(c.Metadata, fname, baseURL, hash) + } + return index, nil +} + +// loadIndex loads an index file and does minimal validity checking. +// +// This will fail if API Version is not set (ErrNoAPIVersion) or if the unmarshal fails. +func loadIndex(data []byte) (*IndexFile, error) { + i := &IndexFile{} + if err := yaml.Unmarshal(data, i); err != nil { + return i, err + } + if i.APIVersion == "" { + // When we leave Beta, we should remove legacy support and just + // return this error: + //return i, ErrNoAPIVersion + return loadUnversionedIndex(data) + } + return i, nil +} + +// unversionedEntry represents a deprecated pre-Alpha.5 format. +// +// This will be removed prior to v2.0.0 +type unversionedEntry struct { + Checksum string `json:"checksum"` + URL string `json:"url"` + Chartfile *chart.Metadata `json:"chartfile"` +} + +// loadUnversionedIndex loads a pre-Alpha.5 index.yaml file. +// +// This format is deprecated. This function will be removed prior to v2.0.0. +func loadUnversionedIndex(data []byte) (*IndexFile, error) { + fmt.Fprintln(os.Stderr, "WARNING: Deprecated index file format. Try 'helm repo update'") + i := map[string]unversionedEntry{} + + // This gets around an error in the YAML parser. Instead of parsing as YAML, + // we convert to JSON, and then decode again. + var err error + data, err = yaml.YAMLToJSON(data) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &i); err != nil { + return nil, err + } + + if len(i) == 0 { + return nil, ErrNoAPIVersion + } + ni := NewIndexFile() + for n, item := range i { + if item.Chartfile == nil || item.Chartfile.Name == "" { + parts := strings.Split(n, "-") + ver := "" + if len(parts) > 1 { + ver = strings.TrimSuffix(parts[1], ".tgz") + } + item.Chartfile = &chart.Metadata{Name: parts[0], Version: ver} + } + ni.Add(item.Chartfile, item.URL, "", item.Checksum) + } + return ni, nil +} diff --git a/pkg/repo/index_test.go b/pkg/repo/index_test.go index 7ba4da6c5..2f23950f1 100644 --- a/pkg/repo/index_test.go +++ b/pkg/repo/index_test.go @@ -33,7 +33,7 @@ const ( ) func TestIndexFile(t *testing.T) { - i := NewChartRepositoryIndex() + i := NewIndexFile() i.Add(&chart.Metadata{Name: "clipper", Version: "0.1.0"}, "clipper-0.1.0.tgz", "http://example.com/charts", "sha256:1234567890") i.Add(&chart.Metadata{Name: "cutter", Version: "0.1.1"}, "cutter-0.1.1.tgz", "http://example.com/charts", "sha256:1234567890abc") i.Add(&chart.Metadata{Name: "cutter", Version: "0.1.0"}, "cutter-0.1.0.tgz", "http://example.com/charts", "sha256:1234567890abc") @@ -75,7 +75,7 @@ func TestLoadIndex(t *testing.T) { } func TestLoadIndexFile(t *testing.T) { - i, err := NewChartRepositoryIndexFromFile(testfile) + i, err := LoadIndexFile(testfile) if err != nil { t.Fatal(err) } @@ -83,13 +83,13 @@ func TestLoadIndexFile(t *testing.T) { } func TestMerge(t *testing.T) { - ind1 := NewChartRepositoryIndex() + ind1 := NewIndexFile() ind1.Add(&chart.Metadata{ Name: "dreadnought", Version: "0.1.0", }, "dreadnought-0.1.0.tgz", "http://example.com", "aaaa") - ind2 := NewChartRepositoryIndex() + ind2 := NewIndexFile() ind2.Add(&chart.Metadata{ Name: "dreadnought", Version: "0.2.0", @@ -133,7 +133,7 @@ func TestDownloadIndexFile(t *testing.T) { defer os.RemoveAll(dirName) indexFilePath := filepath.Join(dirName, testRepo+"-index.yaml") - r, err := NewChartRepository(&ChartRepositoryConfig{ + r, err := NewChartRepository(&Entry{ Name: testRepo, URL: srv.URL, Cache: indexFilePath, @@ -164,7 +164,7 @@ func TestDownloadIndexFile(t *testing.T) { verifyLocalIndex(t, i) } -func verifyLocalIndex(t *testing.T, i *ChartRepositoryIndex) { +func verifyLocalIndex(t *testing.T, i *IndexFile) { numEntries := len(i.Entries) if numEntries != 2 { t.Errorf("Expected 2 entries in index file but got %d", numEntries) @@ -264,7 +264,7 @@ func verifyLocalIndex(t *testing.T, i *ChartRepositoryIndex) { func TestIndexDirectory(t *testing.T) { dir := "testdata/repository" - index, err := NewChartRepositoryIndexFromDirectory(dir, "http://localhost:8080") + index, err := IndexDirectory(dir, "http://localhost:8080") if err != nil { t.Fatal(err) } @@ -314,7 +314,7 @@ func TestLoadUnversionedIndex(t *testing.T) { } func TestIndexAdd(t *testing.T) { - i := NewChartRepositoryIndex() + i := NewIndexFile() i.Add(&chart.Metadata{Name: "clipper", Version: "0.1.0"}, "clipper-0.1.0.tgz", "http://example.com/charts", "sha256:1234567890") if i.Entries["clipper"][0].URLs[0] != "http://example.com/charts/clipper-0.1.0.tgz" { diff --git a/pkg/repo/local.go b/pkg/repo/local.go index 932811be2..f13a4d0ac 100644 --- a/pkg/repo/local.go +++ b/pkg/repo/local.go @@ -82,7 +82,7 @@ func (s *RepositoryServer) htmlIndex(w http.ResponseWriter, r *http.Request) { t := htemplate.Must(htemplate.New("index.html").Parse(indexHTMLTemplate)) // load index lrp := filepath.Join(s.RepoPath, "index.yaml") - i, err := NewChartRepositoryIndexFromFile(lrp) + i, err := LoadIndexFile(lrp) if err != nil { http.Error(w, err.Error(), 500) return @@ -107,7 +107,7 @@ func AddChartToLocalRepo(ch *chart.Chart, path string) error { // Reindex adds an entry to the index file at the given path func Reindex(ch *chart.Chart, path string) error { name := ch.Metadata.Name + "-" + ch.Metadata.Version - y, err := NewChartRepositoryIndexFromFile(path) + y, err := LoadIndexFile(path) if err != nil { return err } diff --git a/pkg/repo/repo.go b/pkg/repo/repo.go index bfc7d4c1d..cbf54c572 100644 --- a/pkg/repo/repo.go +++ b/pkg/repo/repo.go @@ -30,35 +30,35 @@ import ( // is fixable. var ErrRepoOutOfDate = errors.New("repository file is out of date") -// RepositoryFile represents the repositories.yaml file in $HELM_HOME -type RepositoryFile struct { - APIVersion string `json:"apiVersion"` - Generated time.Time `json:"generated"` - Repositories []*ChartRepositoryConfig `json:"repositories"` +// RepoFile represents the repositories.yaml file in $HELM_HOME +type RepoFile struct { + APIVersion string `json:"apiVersion"` + Generated time.Time `json:"generated"` + Repositories []*Entry `json:"repositories"` } -// NewRepositoryFile generates an empty repositories file. +// NewRepoFile generates an empty repositories file. // // Generated and APIVersion are automatically set. -func NewRepositoryFile() *RepositoryFile { - return &RepositoryFile{ +func NewRepoFile() *RepoFile { + return &RepoFile{ APIVersion: APIVersionV1, Generated: time.Now(), - Repositories: []*ChartRepositoryConfig{}, + Repositories: []*Entry{}, } } -// LoadRepositoryFile takes a file at the given path and returns a RepositoryFile object +// LoadRepositoriesFile takes a file at the given path and returns a RepoFile object // -// If this returns ErrRepoOutOfDate, it also returns a recovered RepositoryFile that +// If this returns ErrRepoOutOfDate, it also returns a recovered RepoFile that // can be saved as a replacement to the out of date file. -func LoadRepositoryFile(path string) (*RepositoryFile, error) { +func LoadRepositoriesFile(path string) (*RepoFile, error) { b, err := ioutil.ReadFile(path) if err != nil { return nil, err } - r := &RepositoryFile{} + r := &RepoFile{} err = yaml.Unmarshal(b, r) if err != nil { return nil, err @@ -70,9 +70,9 @@ func LoadRepositoryFile(path string) (*RepositoryFile, error) { if err = yaml.Unmarshal(b, &m); err != nil { return nil, err } - r := NewRepositoryFile() + r := NewRepoFile() for k, v := range m { - r.Add(&ChartRepositoryConfig{ + r.Add(&Entry{ Name: k, URL: v, Cache: fmt.Sprintf("%s-index.yaml", k), @@ -85,13 +85,13 @@ func LoadRepositoryFile(path string) (*RepositoryFile, error) { } // Add adds one or more repo entries to a repo file. -func (r *RepositoryFile) Add(re ...*ChartRepositoryConfig) { +func (r *RepoFile) Add(re ...*Entry) { r.Repositories = append(r.Repositories, re...) } // Update attempts to replace one or more repo entries in a repo file. If an // entry with the same name doesn't exist in the repo file it will add it. -func (r *RepositoryFile) Update(re ...*ChartRepositoryConfig) { +func (r *RepoFile) Update(re ...*Entry) { for _, target := range re { found := false for j, repo := range r.Repositories { @@ -108,7 +108,7 @@ func (r *RepositoryFile) Update(re ...*ChartRepositoryConfig) { } // Has returns true if the given name is already a repository name. -func (r *RepositoryFile) Has(name string) bool { +func (r *RepoFile) Has(name string) bool { for _, rf := range r.Repositories { if rf.Name == name { return true @@ -118,8 +118,8 @@ func (r *RepositoryFile) Has(name string) bool { } // Remove removes the entry from the list of repositories. -func (r *RepositoryFile) Remove(name string) bool { - cp := []*ChartRepositoryConfig{} +func (r *RepoFile) Remove(name string) bool { + cp := []*Entry{} found := false for _, rf := range r.Repositories { if rf.Name == name { @@ -133,7 +133,7 @@ func (r *RepositoryFile) Remove(name string) bool { } // WriteFile writes a repositories file to the given path. -func (r *RepositoryFile) WriteFile(path string, perm os.FileMode) error { +func (r *RepoFile) WriteFile(path string, perm os.FileMode) error { data, err := yaml.Marshal(r) if err != nil { return err diff --git a/pkg/repo/repo_test.go b/pkg/repo/repo_test.go index 451e035aa..c0b0e6da7 100644 --- a/pkg/repo/repo_test.go +++ b/pkg/repo/repo_test.go @@ -21,14 +21,14 @@ import "testing" const testRepositoriesFile = "testdata/repositories.yaml" func TestRepoFile(t *testing.T) { - rf := NewRepositoryFile() + rf := NewRepoFile() rf.Add( - &ChartRepositoryConfig{ + &Entry{ Name: "stable", URL: "https://example.com/stable/charts", Cache: "stable-index.yaml", }, - &ChartRepositoryConfig{ + &Entry{ Name: "incubator", URL: "https://example.com/incubator", Cache: "incubator-index.yaml", @@ -59,21 +59,21 @@ func TestRepoFile(t *testing.T) { } func TestNewRepositoriesFile(t *testing.T) { - expects := NewRepositoryFile() + expects := NewRepoFile() expects.Add( - &ChartRepositoryConfig{ + &Entry{ Name: "stable", URL: "https://example.com/stable/charts", Cache: "stable-index.yaml", }, - &ChartRepositoryConfig{ + &Entry{ Name: "incubator", URL: "https://example.com/incubator", Cache: "incubator-index.yaml", }, ) - repofile, err := LoadRepositoryFile(testRepositoriesFile) + repofile, err := LoadRepositoriesFile(testRepositoriesFile) if err != nil { t.Errorf("%q could not be loaded: %s", testRepositoriesFile, err) } @@ -97,7 +97,7 @@ func TestNewRepositoriesFile(t *testing.T) { } func TestNewPreV1RepositoriesFile(t *testing.T) { - r, err := LoadRepositoryFile("testdata/old-repositories.yaml") + r, err := LoadRepositoriesFile("testdata/old-repositories.yaml") if err != nil && err != ErrRepoOutOfDate { t.Fatal(err) } diff --git a/pkg/repo/repotest/server.go b/pkg/repo/repotest/server.go index 07973a692..4bb1f57a9 100644 --- a/pkg/repo/repotest/server.go +++ b/pkg/repo/repotest/server.go @@ -114,7 +114,7 @@ func (s *Server) CopyCharts(origin string) ([]string, error) { // CreateIndex will read docroot and generate an index.yaml file. func (s *Server) CreateIndex() error { // generate the index - index, err := repo.NewChartRepositoryIndexFromDirectory(s.docroot, s.URL()) + index, err := repo.IndexDirectory(s.docroot, s.URL()) if err != nil { return err } @@ -160,8 +160,8 @@ func (s *Server) LinkIndices() error { // setTestingRepository sets up a testing repository.yaml with only the given name/URL. func setTestingRepository(home helmpath.Home, name, url string) error { - r := repo.NewRepositoryFile() - r.Add(&repo.ChartRepositoryConfig{ + r := repo.NewRepoFile() + r.Add(&repo.Entry{ Name: name, URL: url, Cache: home.CacheIndex(name), diff --git a/pkg/repo/repotest/server_test.go b/pkg/repo/repotest/server_test.go index 99917fc3b..1d4c78e41 100644 --- a/pkg/repo/repotest/server_test.go +++ b/pkg/repo/repotest/server_test.go @@ -77,7 +77,7 @@ func TestServer(t *testing.T) { return } - m := repo.NewChartRepositoryIndex() + m := repo.NewIndexFile() if err := yaml.Unmarshal(data, m); err != nil { t.Error(err) return