Merge pull request #745 from michelleN/repo-index

feat(helm): generate index file in given directory with given url

cmd/helm/repo.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"path/filepath"
 
 	"github.com/gosuri/uitable"
 	"github.com/kubernetes/helm/pkg/repo"
@@ -15,6 +16,7 @@ func init() {
 	repoCmd.AddCommand(repoAddCmd)
 	repoCmd.AddCommand(repoListCmd)
 	repoCmd.AddCommand(repoRemoveCmd)
+	repoCmd.AddCommand(repoIndexCmd)
 	RootCommand.AddCommand(repoCmd)
 }
 
@@ -41,6 +43,12 @@ var repoRemoveCmd = &cobra.Command{
 	RunE:  runRepoRemove,
 }
 
+var repoIndexCmd = &cobra.Command{
+	Use:   "index [flags] [DIR] [REPO_URL]",
+	Short: "generate an index file for a chart repository given a directory",
+	RunE:  runRepoIndex,
+}
+
 func runRepoAdd(cmd *cobra.Command, args []string) error {
 	if err := checkArgsLength(2, len(args), "name for the chart repository", "the url of the chart repository"); err != nil {
 		return err
@@ -87,6 +95,35 @@ func runRepoRemove(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func runRepoIndex(cmd *cobra.Command, args []string) error {
+	if err := checkArgsLength(2, len(args), "path to a directory", "url of chart repository"); err != nil {
+		return err
+	}
+
+	path, err := filepath.Abs(args[0])
+	if err != nil {
+		return err
+	}
+
+	if err := index(path, args[1]); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func index(dir, url string) error {
+	chartRepo, err := repo.LoadChartRepository(dir, url)
+	if err != nil {
+		return err
+	}
+
+	if err := chartRepo.Index(); err != nil {
+		return err
+	}
+	return nil
+}
+
 func removeRepoLine(name string) error {
 	r, err := repo.LoadRepositoriesFile(repositoriesFile())
 	if err != nil {

cmd/helm/search.go

@@ -48,7 +48,7 @@ func searchChartRefsForPattern(search string, chartRefs map[string]*repo.ChartRe
 			matches = append(matches, k)
 			continue
 		}
-		for _, keyword := range c.Keywords {
+		for _, keyword := range c.Chartfile.Keywords {
 			if strings.Contains(keyword, search) {
 				matches = append(matches, k)
 			}

cmd/helm/search_test.go

@@ -1,9 +1,9 @@
 package main
 
 import (
-	"github.com/kubernetes/helm/pkg/repo"
-
 	"testing"
+
+	"github.com/kubernetes/helm/pkg/repo"
 )
 
 const testDir = "testdata/"

cmd/helm/testdata/foobar-index.yaml

@@ -1,18 +1,24 @@
 foobar-0.1.0:
   url: http://storage.googleapis.com/kubernetes-charts/nginx-0.1.0.tgz
   name: foobar
-  description: string
+  removed: false
-  version: 0.1.0
+  chartfile:
-  home: https://github.com/foo
+    name: foobar
-  keywords:
+    description: string
-    - dummy
+    version: 0.1.0
-    - hokey
+    home: https://github.com/foo
+    keywords:
+      - dummy
+      - hokey
 oddness-1.2.3:
   url: http://storage.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz
   name: oddness
-  description: string
+  removed: false
-  version: 1.2.3
+  chartfile:
-  home: https://github.com/something
+    name: oddness
-  keywords:
+    description: string
-    - duck
+    version: 1.2.3
-    - sumtin
+    home: https://github.com/something
+    keywords:
+      - duck
+      - sumtin

cmd/helm/testdata/local-index.yaml

@@ -1,22 +1,28 @@
 nginx-0.1.0:
   url: http://storage.googleapis.com/kubernetes-charts/nginx-0.1.0.tgz
   name: nginx
-  description: string
+  removed: false
-  version: 0.1.0
+  chartfile:
-  home: https://github.com/something
+    name: nginx
-  keywords:
+    description: string
-    - popular
+    version: 0.1.0
-    - web server
+    home: https://github.com/something
-    - proxy
+    keywords:
+      - popular
+      - web server
+      - proxy
 alpine-1.0.0:
   url: http://storage.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz
   name: alpine
-  description: string
+  removed: false
-  version: 1.0.0
+  chartfile:
-  home: https://github.com/something
+    name: alpine
-  keywords:
+    description: string
-    - linux
+    version: 1.0.0
-    - alpine
+    home: https://github.com/something
-    - small
+    keywords:
-    - sumtin
+      - linux
+      - alpine
+      - small
+      - sumtin
 

pkg/repo/index.go

@@ -6,8 +6,12 @@ import (
 	"strings"
 
 	"gopkg.in/yaml.v2"
+
+	"github.com/kubernetes/helm/pkg/chart"
 )
 
+var indexPath = "index.yaml"
+
 // IndexFile represents the index file in a chart repository
 type IndexFile struct {
 	Entries map[string]*ChartRef
@@ -15,10 +19,12 @@ type IndexFile struct {
 
 // ChartRef represents a chart entry in the IndexFile
 type ChartRef struct {
-	Name     string   `yaml:"name"`
+	Name      string          `yaml:"name"`
-	URL      string   `yaml:"url"`
+	URL       string          `yaml:"url"`
-	Keywords []string `yaml:"keywords"`
+	Created   string          `yaml:"created,omitempty"`
-	Removed  bool     `yaml:"removed,omitempty"`
+	Removed   bool            `yaml:"removed,omitempty"`
+	Digest    string          `yaml:"digest,omitempty"`
+	Chartfile chart.Chartfile `yaml:"chartfile"`
 }
 
 // DownloadIndexFile uses
@@ -49,3 +55,45 @@ func DownloadIndexFile(repoName, url, indexFileName string) error {
 
 	return nil
 }
+
+// UnmarshalYAML unmarshals the index file
+func (i *IndexFile) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var refs map[string]*ChartRef
+	if err := unmarshal(&refs); err != nil {
+		if _, ok := err.(*yaml.TypeError); !ok {
+			return err
+		}
+	}
+	i.Entries = refs
+	return nil
+}
+
+func (i *IndexFile) addEntry(name string, url string) ([]byte, error) {
+	if i.Entries == nil {
+		i.Entries = make(map[string]*ChartRef)
+	}
+	entry := ChartRef{Name: name, URL: url}
+	i.Entries[name] = &entry
+	out, err := yaml.Marshal(&i.Entries)
+	if err != nil {
+		return nil, err
+	}
+
+	return out, nil
+}
+
+// LoadIndexFile takes a file at the given path and returns an IndexFile object
+func LoadIndexFile(path string) (*IndexFile, error) {
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var indexfile IndexFile
+	err = yaml.Unmarshal(b, &indexfile)
+	if err != nil {
+		return nil, err
+	}
+
+	return &indexfile, nil
+}

pkg/repo/index_test.go

@@ -12,6 +12,8 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+const testfile = "testdata/local-index.yaml"
+
 var (
 	testRepo = "test-repo"
 )
@@ -52,5 +54,40 @@ func TestDownloadIndexFile(t *testing.T) {
 		t.Errorf("Expected 2 entries in index file but got %v", numEntries)
 	}
 	os.Remove(path)
+	os.Remove(dirName)
+
+}
 
+func TestLoadIndexFile(t *testing.T) {
+	cf, err := LoadIndexFile(testfile)
+	if err != nil {
+		t.Errorf("Failed to load index file: %s", err)
+	}
+	if len(cf.Entries) != 2 {
+		t.Errorf("Expected 2 entries in the index file, but got %d", len(cf.Entries))
+	}
+	nginx := false
+	alpine := false
+	for k, e := range cf.Entries {
+		if k == "nginx-0.1.0" {
+			if e.Name == "nginx" {
+				if len(e.Chartfile.Keywords) == 3 {
+					nginx = true
+				}
+			}
+		}
+		if k == "alpine-1.0.0" {
+			if e.Name == "alpine" {
+				if len(e.Chartfile.Keywords) == 4 {
+					alpine = true
+				}
+			}
+		}
+	}
+	if !nginx {
+		t.Errorf("nginx entry was not decoded properly")
+	}
+	if !alpine {
+		t.Errorf("alpine entry was not decoded properly")
+	}
 }

pkg/repo/local.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 
 	"github.com/kubernetes/helm/pkg/chart"
-	"gopkg.in/yaml.v2"
 )
 
 var localRepoPath string
@@ -54,22 +53,6 @@ func AddChartToLocalRepo(ch *chart.Chart, path string) error {
 	return nil
 }
 
-// LoadIndexFile takes a file at the given path and returns an IndexFile object
-func LoadIndexFile(path string) (*IndexFile, error) {
-	b, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-
-	//TODO: change variable name - y is not helpful :P
-	var y IndexFile
-	err = yaml.Unmarshal(b, &y)
-	if err != nil {
-		return nil, err
-	}
-	return &y, nil
-}
-
 // Reindex adds an entry to the index file at the given path
 func Reindex(ch *chart.Chart, path string) error {
 	name := ch.Chartfile().Name + "-" + ch.Chartfile().Version
@@ -87,7 +70,7 @@ func Reindex(ch *chart.Chart, path string) error {
 	if !found {
 		url := "localhost:8879/charts/" + name + ".tgz"
 
-		out, err := y.insertChartEntry(name, url)
+		out, err := y.addEntry(name, url)
 		if err != nil {
 			return err
 		}
@@ -96,29 +79,3 @@ func Reindex(ch *chart.Chart, path string) error {
 	}
 	return nil
 }
-
-// UnmarshalYAML unmarshals the index file
-func (i *IndexFile) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	var refs map[string]*ChartRef
-	if err := unmarshal(&refs); err != nil {
-		if _, ok := err.(*yaml.TypeError); !ok {
-			return err
-		}
-	}
-	i.Entries = refs
-	return nil
-}
-
-func (i *IndexFile) insertChartEntry(name string, url string) ([]byte, error) {
-	if i.Entries == nil {
-		i.Entries = make(map[string]*ChartRef)
-	}
-	entry := ChartRef{Name: name, URL: url}
-	i.Entries[name] = &entry
-	out, err := yaml.Marshal(&i.Entries)
-	if err != nil {
-		return nil, err
-	}
-
-	return out, nil
-}

pkg/repo/local_test.go

@@ -1,41 +0,0 @@
-package repo
-
-import (
-	"testing"
-)
-
-const testfile = "testdata/local-index.yaml"
-
-func TestLoadIndexFile(t *testing.T) {
-	cf, err := LoadIndexFile(testfile)
-	if err != nil {
-		t.Errorf("Failed to load index file: %s", err)
-	}
-	if len(cf.Entries) != 2 {
-		t.Errorf("Expected 2 entries in the index file, but got %d", len(cf.Entries))
-	}
-	nginx := false
-	alpine := false
-	for k, e := range cf.Entries {
-		if k == "nginx-0.1.0" {
-			if e.Name == "nginx" {
-				if len(e.Keywords) == 3 {
-					nginx = true
-				}
-			}
-		}
-		if k == "alpine-1.0.0" {
-			if e.Name == "alpine" {
-				if len(e.Keywords) == 4 {
-					alpine = true
-				}
-			}
-		}
-	}
-	if !nginx {
-		t.Errorf("nginx entry was not decoded properly")
-	}
-	if !alpine {
-		t.Errorf("alpine entry was not decoded properly")
-	}
-}

pkg/repo/repo.go

@@ -1,12 +1,29 @@
 package repo
 
 import (
+	"crypto/sha1"
+	"errors"
+	"fmt"
 	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
 
 	"gopkg.in/yaml.v2"
+
+	"github.com/kubernetes/helm/pkg/chart"
 )
 
-// RepoFile represents the .repositories file in $HELM_HOME
+// ChartRepository represents a chart repository
+type ChartRepository struct {
+	RootPath   string
+	URL        string // URL of repository
+	ChartPaths []string
+	IndexFile  *IndexFile
+}
+
+// RepoFile represents the repositories.yaml file in $HELM_HOME
 type RepoFile struct {
 	Repositories map[string]string
 }
@@ -38,3 +55,111 @@ func (rf *RepoFile) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	rf.Repositories = repos
 	return nil
 }
+
+// LoadChartRepository takes in a path to a local chart repository
+//      which contains packaged charts and an index.yaml file
+//
+// This function evaluates the contents of the directory and
+// returns a ChartRepository
+func LoadChartRepository(dir, url string) (*ChartRepository, error) {
+	dirInfo, err := os.Stat(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	if !dirInfo.IsDir() {
+		return nil, errors.New(dir + "is not a directory")
+	}
+
+	r := &ChartRepository{RootPath: dir, URL: url}
+
+	filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
+		if !f.IsDir() {
+			if strings.Contains(f.Name(), "index.yaml") {
+				i, err := LoadIndexFile(path)
+				if err != nil {
+					return nil
+				}
+				r.IndexFile = i
+			} else {
+				// TODO: check for tgz extension
+				r.ChartPaths = append(r.ChartPaths, path)
+			}
+		}
+		return nil
+	})
+
+	return r, nil
+}
+
+func (r *ChartRepository) saveIndexFile() error {
+	index, err := yaml.Marshal(&r.IndexFile.Entries)
+	if err != nil {
+		return err
+	}
+
+	if err = ioutil.WriteFile(filepath.Join(r.RootPath, indexPath), index, 0644); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *ChartRepository) Index() error {
+	if r.IndexFile == nil {
+		r.IndexFile = &IndexFile{Entries: make(map[string]*ChartRef)}
+	}
+
+	for _, path := range r.ChartPaths {
+		ch, err := chart.Load(path)
+		if err != nil {
+			return err
+		}
+
+		chartfile := ch.Chartfile()
+		hash, err := generateDigest(path)
+		if err != nil {
+			return err
+		}
+
+		key := chartfile.Name + "-" + chartfile.Version
+		if r.IndexFile.Entries == nil {
+			r.IndexFile.Entries = make(map[string]*ChartRef)
+		}
+
+		ref, ok := r.IndexFile.Entries[key]
+		var created string
+		if ok && ref.Created != "" {
+			created = ref.Created
+		} else {
+			created = time.Now().UTC().String()
+		}
+
+		entry := &ChartRef{Chartfile: *chartfile, Name: chartfile.Name, URL: r.URL, Created: created, Digest: hash, Removed: false}
+
+		r.IndexFile.Entries[key] = entry
+
+	}
+
+	if err := r.saveIndexFile(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func generateDigest(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+
+	b, err := ioutil.ReadAll(f)
+	if err != nil {
+		return "", err
+	}
+
+	result := sha1.Sum(b)
+
+	return fmt.Sprintf("%x", result), nil
+}

pkg/repo/repo_test.go

@@ -0,0 +1,120 @@
+package repo
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+)
+
+const testRepositoriesFile = "testdata/repositories.yaml"
+const testRepository = "testdata/repository"
+const testURL = "http://example-charts.com"
+
+func TestLoadChartRepository(t *testing.T) {
+	cr, err := LoadChartRepository(testRepository, testURL)
+	if err != nil {
+		t.Errorf("Problem loading chart repository from %s: %v", testRepository, err)
+	}
+
+	paths := []string{filepath.Join(testRepository, "frobnitz-1.2.3.tgz"), filepath.Join(testRepository, "sprocket-1.2.0.tgz")}
+
+	if cr.RootPath != testRepository {
+		t.Errorf("Expected %s as RootPath but got %s", testRepository, cr.RootPath)
+	}
+
+	if !reflect.DeepEqual(cr.ChartPaths, paths) {
+		t.Errorf("Expected %#v but got %#v\n", paths, cr.ChartPaths)
+	}
+
+	if cr.URL != testURL {
+		t.Errorf("Expected url for chart repository to be %s but got %s", testURL, cr.URL)
+	}
+}
+
+func TestIndex(t *testing.T) {
+	cr, err := LoadChartRepository(testRepository, testURL)
+	if err != nil {
+		t.Errorf("Problem loading chart repository from %s: %v", testRepository, err)
+	}
+
+	err = cr.Index()
+	if err != nil {
+		t.Errorf("Error performing index: %v\n", err)
+	}
+
+	tempIndexPath := filepath.Join(testRepository, indexPath)
+	actual, err := LoadIndexFile(tempIndexPath)
+	defer os.Remove(tempIndexPath) // clean up
+	if err != nil {
+		t.Errorf("Error loading index file %v", err)
+	}
+
+	entries := actual.Entries
+	numEntries := len(entries)
+	if numEntries != 2 {
+		t.Errorf("Expected 2 charts to be listed in index file but got %v", numEntries)
+	}
+
+	timestamps := make(map[string]string)
+	var empty time.Time
+	for chartName, details := range entries {
+		if details == nil {
+			t.Errorf("Chart Entry is not filled out for %s", chartName)
+		}
+
+		if details.Created == empty.String() {
+			t.Errorf("Created timestamp under %s chart entry is nil", chartName)
+		}
+		timestamps[chartName] = details.Created
+
+		if details.Digest == "" {
+			t.Errorf("Digest was not set for %s", chartName)
+		}
+	}
+
+	if err = cr.Index(); err != nil {
+		t.Errorf("Error performing index the second time: %v\n", err)
+	}
+	second, err := LoadIndexFile(tempIndexPath)
+	if err != nil {
+		t.Errorf("Error loading index file second time: %#v\n", err)
+	}
+
+	for chart, created := range timestamps {
+		v, ok := second.Entries[chart]
+		if !ok {
+			t.Errorf("Expected %s chart entry in index file but did not find it", chart)
+		}
+		if v.Created != created {
+			t.Errorf("Expected Created timestamp to be %s, but got %s for chart %s", created, v.Created, chart)
+		}
+	}
+}
+
+func TestLoadRepositoriesFile(t *testing.T) {
+	rf, err := LoadRepositoriesFile(testRepositoriesFile)
+	if err != nil {
+		t.Errorf(testRepositoriesFile + " could not be loaded: " + err.Error())
+	}
+	expected := map[string]string{"best-charts-ever": "http://best-charts-ever.com",
+		"okay-charts": "http://okay-charts.org", "example123": "http://examplecharts.net/charts/123"}
+
+	numOfRepositories := len(rf.Repositories)
+	expectedNumOfRepositories := 3
+	if numOfRepositories != expectedNumOfRepositories {
+		t.Errorf("Expected %v repositories but only got %v", expectedNumOfRepositories, numOfRepositories)
+	}
+
+	for expectedRepo, expectedURL := range expected {
+		actual, ok := rf.Repositories[expectedRepo]
+		if !ok {
+			t.Errorf("Expected repository: %v but was not found", expectedRepo)
+		}
+
+		if expectedURL != actual {
+			t.Errorf("Expected url %s for the %s repository but got %s ", expectedURL, expectedRepo, actual)
+		}
+	}
+}

pkg/repo/testdata/local-index.yaml

@@ -1,22 +1,26 @@
 nginx-0.1.0:
   url: http://storage.googleapis.com/kubernetes-charts/nginx-0.1.0.tgz
   name: nginx
-  description: string
+  chartfile:
-  version: 0.1.0
+      name: nginx
-  home: https://github.com/something
+      description: string
-  keywords:
+      version: 0.1.0
-    - popular
+      home: https://github.com/something
-    - web server
+      keywords:
-    - proxy
+        - popular
+        - web server
+        - proxy
 alpine-1.0.0:
   url: http://storage.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz
   name: alpine
-  description: string
+  chartfile:
-  version: 1.0.0
+      name: alpine
-  home: https://github.com/something
+      description: string
-  keywords:
+      version: 1.0.0
-    - linux
+      home: https://github.com/something
-    - alpine
+      keywords:
-    - small
+        - linux
-    - sumtin
+        - alpine
+        - small
+        - sumtin
 

pkg/repo/testdata/repositories.yaml

@@ -0,0 +1,3 @@
+best-charts-ever: http://best-charts-ever.com
+okay-charts: http://okay-charts.org
+example123: http://examplecharts.net/charts/123