fix(repo): auto-update index file formats

This performs a relatively weak in-memory translation of index file
data. It does not, in most cases, write the corrected data to disk, and
it emits a warning directly to STDERR each time it loads a deprecated
index.

Known limitations:

- It cannot recover certain bogus records that earlier alpha releases
  generated (notably, where all chartfile data is missing)
- In some cases, it has to parse a filename to get version info. This is
  lossy.
- Because it takes three passes through the YAML and JSON unmarshal, it
  is not performant.

This feature is transitional and should be removed during the Beta
cycle, prior to the release of 2.0.0.

Closes #1265

cmd/helm/init.go

@@ -138,7 +138,7 @@ func ensureHome(home helmpath.Home, out io.Writer) error {
 		}
 		cif := home.CacheIndex(stableRepository)
 		if err := repo.DownloadIndexFile(stableRepository, stableRepositoryURL, cif); err != nil {
-			fmt.Fprintf(out, "WARNING: Failed to download %s: %s (run 'helm update')\n", stableRepository, err)
+			fmt.Fprintf(out, "WARNING: Failed to download %s: %s (run 'helm repo update')\n", stableRepository, err)
 		}
 	} else if fi.IsDir() {
 		return fmt.Errorf("%s must be a file, not a directory", repoFile)

cmd/helm/search.go

@@ -106,7 +106,7 @@ func (s *searchCmd) buildIndex() (*search.Index, error) {
 		f := s.helmhome.CacheIndex(n)
 		ind, err := repo.LoadIndexFile(f)
 		if err != nil {
-			fmt.Fprintf(s.out, "WARNING: Repo %q is corrupt or missing. Try 'helm update':\n\t%s\n", f, err)
+			fmt.Fprintf(s.out, "WARNING: Repo %q is corrupt or missing. Try 'helm repo update':\n\t%s\n", f, err)
 			continue
 		}
 

pkg/repo/index.go

@@ -17,7 +17,9 @@ limitations under the License.
 package repo
 
 import (
+	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -39,8 +41,14 @@ var indexPath = "index.yaml"
 // APIVersionV1 is the v1 API version for index and repository files.
 const APIVersionV1 = "v1"
 
-// ErrNoAPIVersion indicates that an API version was not specified.
+var (
-var ErrNoAPIVersion = errors.New("no API version specified")
+	// ErrNoAPIVersion indicates that an API version was not specified.
+	ErrNoAPIVersion = errors.New("no API version specified")
+	// ErrNoChartVersion indicates that a chart with the given version is not found.
+	ErrNoChartVersion = errors.New("no chart version found")
+	// ErrNoChartName indicates that a chart with the given name is not found.
+	ErrNoChartName = errors.New("no chart name found")
+)
 
 // ChartVersions is a list of versioned chart references.
 // Implements a sorter on Version.
@@ -86,8 +94,12 @@ func NewIndexFile() *IndexFile {
 
 // Add adds a file to the index
 func (i IndexFile) Add(md *chart.Metadata, filename, baseURL, digest string) {
+	u := filename
+	if baseURL != "" {
+		u = baseURL + "/" + filename
+	}
 	cr := &ChartVersion{
-		URLs:     []string{baseURL + "/" + filename},
+		URLs:     []string{u},
 		Metadata: md,
 		Digest:   digest,
 		Created:  time.Now(),
@@ -101,17 +113,8 @@ func (i IndexFile) Add(md *chart.Metadata, filename, baseURL, digest string) {
 
 // Has returns true if the index has an entry for a chart with the given name and exact version.
 func (i IndexFile) Has(name, version string) bool {
-	vs, ok := i.Entries[name]
+	_, err := i.Get(name, version)
-	if !ok {
+	return err == nil
-		return false
-	}
-	for _, ver := range vs {
-		// TODO: Do we need to normalize the version field with the SemVer lib?
-		if ver.Version == version {
-			return true
-		}
-	}
-	return false
 }
 
 // SortEntries sorts the entries by version in descending order.
@@ -126,6 +129,26 @@ func (i IndexFile) SortEntries() {
 	}
 }
 
+// Get returns the ChartVersion for the given name.
+//
+// If version is empty, this will return the chart with the highest version.
+func (i IndexFile) Get(name, version string) (*ChartVersion, error) {
+	vs, ok := i.Entries[name]
+	if !ok {
+		return nil, ErrNoChartName
+	}
+	if version == "" && len(vs) > 0 {
+		return vs[0], nil
+	}
+	for _, ver := range vs {
+		// TODO: Do we need to normalize the version field with the SemVer lib?
+		if ver.Version == version {
+			return ver, nil
+		}
+	}
+	return nil, ErrNoChartVersion
+}
+
 // WriteFile writes an index file to the given destination path.
 //
 // The mode on the file is set to 'mode'.
@@ -207,11 +230,59 @@ func LoadIndex(data []byte) (*IndexFile, error) {
 		return i, err
 	}
 	if i.APIVersion == "" {
-		return i, ErrNoAPIVersion
+		// When we leave Beta, we should remove legacy support and just
+		// return this error:
+		//return i, ErrNoAPIVersion
+		return loadUnversionedIndex(data)
 	}
 	return i, nil
 }
 
+// unversionedEntry represents a deprecated pre-Alpha.5 format.
+//
+// This will be removed prior to v2.0.0
+type unversionedEntry struct {
+	Checksum  string          `json:"checksum"`
+	URL       string          `json:"url"`
+	Chartfile *chart.Metadata `json:"chartfile"`
+}
+
+// loadUnversionedIndex loads a pre-Alpha.5 index.yaml file.
+//
+// This format is deprecated. This function will be removed prior to v2.0.0.
+func loadUnversionedIndex(data []byte) (*IndexFile, error) {
+	fmt.Fprintln(os.Stderr, "WARNING: Deprecated index file format. Try 'helm repo update'")
+	i := map[string]unversionedEntry{}
+
+	// This gets around an error in the YAML parser. Instead of parsing as YAML,
+	// we convert to JSON, and then decode again.
+	var err error
+	data, err = yaml.YAMLToJSON(data)
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(data, &i); err != nil {
+		return nil, err
+	}
+
+	if len(i) == 0 {
+		return nil, ErrNoAPIVersion
+	}
+	ni := NewIndexFile()
+	for n, item := range i {
+		if item.Chartfile == nil || item.Chartfile.Name == "" {
+			parts := strings.Split(n, "-")
+			ver := ""
+			if len(parts) > 1 {
+				ver = strings.TrimSuffix(parts[1], ".tgz")
+			}
+			item.Chartfile = &chart.Metadata{Name: parts[0], Version: ver}
+		}
+		ni.Add(item.Chartfile, item.URL, "", item.Checksum)
+	}
+	return ni, nil
+}
+
 // LoadIndexFile takes a file at the given path and returns an IndexFile object
 func LoadIndexFile(path string) (*IndexFile, error) {
 	b, err := ioutil.ReadFile(path)

pkg/repo/index_test.go

@@ -250,3 +250,23 @@ func TestIndexDirectory(t *testing.T) {
 		t.Errorf("Expected frobnitz, got %q", frob.Name)
 	}
 }
+
+func TestLoadUnversionedIndex(t *testing.T) {
+	data, err := ioutil.ReadFile("testdata/unversioned-index.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ind, err := loadUnversionedIndex(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if l := len(ind.Entries); l != 2 {
+		t.Fatalf("Expected 2 entries, got %d", l)
+	}
+
+	if l := len(ind.Entries["mysql"]); l != 3 {
+		t.Fatalf("Expected 3 mysql versions, got %d", l)
+	}
+}

pkg/repo/testdata/unversioned-index.yaml

@@ -0,0 +1,64 @@
+memcached-0.1.0:
+  name: memcached
+  url: https://mumoshu.github.io/charts/memcached-0.1.0.tgz
+  created: 2016-08-04 02:05:02.259205055 +0000 UTC
+  checksum: ce9b76576c4b4eb74286fa30a978c56d69e7a522
+  chartfile:
+    name: memcached
+    home: http://https://hub.docker.com/_/memcached/
+    sources: []
+    version: 0.1.0
+    description: A simple Memcached cluster
+    keywords: []
+    maintainers:
+    - name: Matt Butcher
+      email: mbutcher@deis.com
+    engine: ""
+mysql-0.2.0:
+  name: mysql
+  url: https://mumoshu.github.io/charts/mysql-0.2.0.tgz
+  created: 2016-08-04 00:42:47.517342022 +0000 UTC
+  checksum: aa5edd2904d639b0b6295f1c7cf4c0a8e4f77dd3
+  chartfile:
+    name: mysql
+    home: https://www.mysql.com/
+    sources: []
+    version: 0.2.0
+    description: Chart running MySQL.
+    keywords: []
+    maintainers:
+    - name: Matt Fisher
+      email: mfisher@deis.com
+    engine: ""
+mysql-0.2.1:
+  name: mysql
+  url: https://mumoshu.github.io/charts/mysql-0.2.1.tgz
+  created: 2016-08-04 02:40:29.717829534 +0000 UTC
+  checksum: 9d9f056171beefaaa04db75680319ca4edb6336a
+  chartfile:
+    name: mysql
+    home: https://www.mysql.com/
+    sources: []
+    version: 0.2.1
+    description: Chart running MySQL.
+    keywords: []
+    maintainers:
+    - name: Matt Fisher
+      email: mfisher@deis.com
+    engine: ""
+mysql-0.2.2:
+  name: mysql
+  url: https://mumoshu.github.io/charts/mysql-0.2.2.tgz
+  created: 2016-08-04 02:40:29.71841952 +0000 UTC
+  checksum: 6d6810e76a5987943faf0040ec22990d9fb141c7
+  chartfile:
+    name: mysql
+    home: https://www.mysql.com/
+    sources: []
+    version: 0.2.2
+    description: Chart running MySQL.
+    keywords: []
+    maintainers:
+    - name: Matt Fisher
+      email: mfisher@deis.com
+    engine: ""