From ea0e665f84cfe63cf89d8690ee738e1ff005536c Mon Sep 17 00:00:00 2001 From: Matt Butcher Date: Tue, 4 Oct 2016 23:11:58 -0600 Subject: [PATCH] fix(repo): auto-update index file formats This performs a relatively weak in-memory translation of index file data. It does not, in most cases, write the corrected data to disk, and it emits a warning directly to STDERR each time it loads a deprecated index. Known limitations: - It cannot recover certain bogus records that earlier alpha releases generated (notably, where all chartfile data is missing) - In some cases, it has to parse a filename to get version info. This is lossy. - Because it takes three passes through the YAML and JSON unmarshal, it is not performant. This feature is transitional and should be removed during the Beta cycle, prior to the release of 2.0.0. Closes #1265 --- cmd/helm/init.go | 2 +- cmd/helm/search.go | 2 +- pkg/repo/index.go | 101 +++++++++++++++++++---- pkg/repo/index_test.go | 20 +++++ pkg/repo/testdata/unversioned-index.yaml | 64 ++++++++++++++ 5 files changed, 172 insertions(+), 17 deletions(-) create mode 100644 pkg/repo/testdata/unversioned-index.yaml diff --git a/cmd/helm/init.go b/cmd/helm/init.go index c0e091041..1546860c6 100644 --- a/cmd/helm/init.go +++ b/cmd/helm/init.go @@ -138,7 +138,7 @@ func ensureHome(home helmpath.Home, out io.Writer) error { } cif := home.CacheIndex(stableRepository) if err := repo.DownloadIndexFile(stableRepository, stableRepositoryURL, cif); err != nil { - fmt.Fprintf(out, "WARNING: Failed to download %s: %s (run 'helm update')\n", stableRepository, err) + fmt.Fprintf(out, "WARNING: Failed to download %s: %s (run 'helm repo update')\n", stableRepository, err) } } else if fi.IsDir() { return fmt.Errorf("%s must be a file, not a directory", repoFile) diff --git a/cmd/helm/search.go b/cmd/helm/search.go index f8b02cff8..61ee4bb15 100644 --- a/cmd/helm/search.go +++ b/cmd/helm/search.go @@ -106,7 +106,7 @@ func (s *searchCmd) buildIndex() (*search.Index, error) { f := s.helmhome.CacheIndex(n) ind, err := repo.LoadIndexFile(f) if err != nil { - fmt.Fprintf(s.out, "WARNING: Repo %q is corrupt or missing. Try 'helm update':\n\t%s\n", f, err) + fmt.Fprintf(s.out, "WARNING: Repo %q is corrupt or missing. Try 'helm repo update':\n\t%s\n", f, err) continue } diff --git a/pkg/repo/index.go b/pkg/repo/index.go index 42f751955..a8589aa19 100644 --- a/pkg/repo/index.go +++ b/pkg/repo/index.go @@ -17,7 +17,9 @@ limitations under the License. package repo import ( + "encoding/json" "errors" + "fmt" "io/ioutil" "net/http" "os" @@ -39,8 +41,14 @@ var indexPath = "index.yaml" // APIVersionV1 is the v1 API version for index and repository files. const APIVersionV1 = "v1" -// ErrNoAPIVersion indicates that an API version was not specified. -var ErrNoAPIVersion = errors.New("no API version specified") +var ( + // ErrNoAPIVersion indicates that an API version was not specified. + ErrNoAPIVersion = errors.New("no API version specified") + // ErrNoChartVersion indicates that a chart with the given version is not found. + ErrNoChartVersion = errors.New("no chart version found") + // ErrNoChartName indicates that a chart with the given name is not found. + ErrNoChartName = errors.New("no chart name found") +) // ChartVersions is a list of versioned chart references. // Implements a sorter on Version. @@ -86,8 +94,12 @@ func NewIndexFile() *IndexFile { // Add adds a file to the index func (i IndexFile) Add(md *chart.Metadata, filename, baseURL, digest string) { + u := filename + if baseURL != "" { + u = baseURL + "/" + filename + } cr := &ChartVersion{ - URLs: []string{baseURL + "/" + filename}, + URLs: []string{u}, Metadata: md, Digest: digest, Created: time.Now(), @@ -101,17 +113,8 @@ func (i IndexFile) Add(md *chart.Metadata, filename, baseURL, digest string) { // Has returns true if the index has an entry for a chart with the given name and exact version. func (i IndexFile) Has(name, version string) bool { - vs, ok := i.Entries[name] - if !ok { - return false - } - for _, ver := range vs { - // TODO: Do we need to normalize the version field with the SemVer lib? - if ver.Version == version { - return true - } - } - return false + _, err := i.Get(name, version) + return err == nil } // SortEntries sorts the entries by version in descending order. @@ -126,6 +129,26 @@ func (i IndexFile) SortEntries() { } } +// Get returns the ChartVersion for the given name. +// +// If version is empty, this will return the chart with the highest version. +func (i IndexFile) Get(name, version string) (*ChartVersion, error) { + vs, ok := i.Entries[name] + if !ok { + return nil, ErrNoChartName + } + if version == "" && len(vs) > 0 { + return vs[0], nil + } + for _, ver := range vs { + // TODO: Do we need to normalize the version field with the SemVer lib? + if ver.Version == version { + return ver, nil + } + } + return nil, ErrNoChartVersion +} + // WriteFile writes an index file to the given destination path. // // The mode on the file is set to 'mode'. @@ -207,11 +230,59 @@ func LoadIndex(data []byte) (*IndexFile, error) { return i, err } if i.APIVersion == "" { - return i, ErrNoAPIVersion + // When we leave Beta, we should remove legacy support and just + // return this error: + //return i, ErrNoAPIVersion + return loadUnversionedIndex(data) } return i, nil } +// unversionedEntry represents a deprecated pre-Alpha.5 format. +// +// This will be removed prior to v2.0.0 +type unversionedEntry struct { + Checksum string `json:"checksum"` + URL string `json:"url"` + Chartfile *chart.Metadata `json:"chartfile"` +} + +// loadUnversionedIndex loads a pre-Alpha.5 index.yaml file. +// +// This format is deprecated. This function will be removed prior to v2.0.0. +func loadUnversionedIndex(data []byte) (*IndexFile, error) { + fmt.Fprintln(os.Stderr, "WARNING: Deprecated index file format. Try 'helm repo update'") + i := map[string]unversionedEntry{} + + // This gets around an error in the YAML parser. Instead of parsing as YAML, + // we convert to JSON, and then decode again. + var err error + data, err = yaml.YAMLToJSON(data) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &i); err != nil { + return nil, err + } + + if len(i) == 0 { + return nil, ErrNoAPIVersion + } + ni := NewIndexFile() + for n, item := range i { + if item.Chartfile == nil || item.Chartfile.Name == "" { + parts := strings.Split(n, "-") + ver := "" + if len(parts) > 1 { + ver = strings.TrimSuffix(parts[1], ".tgz") + } + item.Chartfile = &chart.Metadata{Name: parts[0], Version: ver} + } + ni.Add(item.Chartfile, item.URL, "", item.Checksum) + } + return ni, nil +} + // LoadIndexFile takes a file at the given path and returns an IndexFile object func LoadIndexFile(path string) (*IndexFile, error) { b, err := ioutil.ReadFile(path) diff --git a/pkg/repo/index_test.go b/pkg/repo/index_test.go index a73cd0352..e5f5256a2 100644 --- a/pkg/repo/index_test.go +++ b/pkg/repo/index_test.go @@ -250,3 +250,23 @@ func TestIndexDirectory(t *testing.T) { t.Errorf("Expected frobnitz, got %q", frob.Name) } } + +func TestLoadUnversionedIndex(t *testing.T) { + data, err := ioutil.ReadFile("testdata/unversioned-index.yaml") + if err != nil { + t.Fatal(err) + } + + ind, err := loadUnversionedIndex(data) + if err != nil { + t.Fatal(err) + } + + if l := len(ind.Entries); l != 2 { + t.Fatalf("Expected 2 entries, got %d", l) + } + + if l := len(ind.Entries["mysql"]); l != 3 { + t.Fatalf("Expected 3 mysql versions, got %d", l) + } +} diff --git a/pkg/repo/testdata/unversioned-index.yaml b/pkg/repo/testdata/unversioned-index.yaml new file mode 100644 index 000000000..7299c66dc --- /dev/null +++ b/pkg/repo/testdata/unversioned-index.yaml @@ -0,0 +1,64 @@ +memcached-0.1.0: + name: memcached + url: https://mumoshu.github.io/charts/memcached-0.1.0.tgz + created: 2016-08-04 02:05:02.259205055 +0000 UTC + checksum: ce9b76576c4b4eb74286fa30a978c56d69e7a522 + chartfile: + name: memcached + home: http://https://hub.docker.com/_/memcached/ + sources: [] + version: 0.1.0 + description: A simple Memcached cluster + keywords: [] + maintainers: + - name: Matt Butcher + email: mbutcher@deis.com + engine: "" +mysql-0.2.0: + name: mysql + url: https://mumoshu.github.io/charts/mysql-0.2.0.tgz + created: 2016-08-04 00:42:47.517342022 +0000 UTC + checksum: aa5edd2904d639b0b6295f1c7cf4c0a8e4f77dd3 + chartfile: + name: mysql + home: https://www.mysql.com/ + sources: [] + version: 0.2.0 + description: Chart running MySQL. + keywords: [] + maintainers: + - name: Matt Fisher + email: mfisher@deis.com + engine: "" +mysql-0.2.1: + name: mysql + url: https://mumoshu.github.io/charts/mysql-0.2.1.tgz + created: 2016-08-04 02:40:29.717829534 +0000 UTC + checksum: 9d9f056171beefaaa04db75680319ca4edb6336a + chartfile: + name: mysql + home: https://www.mysql.com/ + sources: [] + version: 0.2.1 + description: Chart running MySQL. + keywords: [] + maintainers: + - name: Matt Fisher + email: mfisher@deis.com + engine: "" +mysql-0.2.2: + name: mysql + url: https://mumoshu.github.io/charts/mysql-0.2.2.tgz + created: 2016-08-04 02:40:29.71841952 +0000 UTC + checksum: 6d6810e76a5987943faf0040ec22990d9fb141c7 + chartfile: + name: mysql + home: https://www.mysql.com/ + sources: [] + version: 0.2.2 + description: Chart running MySQL. + keywords: [] + maintainers: + - name: Matt Fisher + email: mfisher@deis.com + engine: ""