From 4bb36c89ab3aa45f92d6645f65c905d7630d2186 Mon Sep 17 00:00:00 2001 From: Michelle Noorali Date: Tue, 10 May 2016 15:15:52 -0400 Subject: [PATCH 1/5] feat(helm): generate index file for repository --- cmd/helm/repo.go | 37 +++++ cmd/helm/search.go | 2 +- pkg/repo/local.go | 45 +---- pkg/repo/local_test.go | 41 ----- pkg/repo/repo.go | 155 +++++++++++++++++- pkg/repo/repo_test.go | 114 +++++++++++++ pkg/repo/testdata/local-index.yaml | 34 ++-- pkg/repo/testdata/repositories.yaml | 3 + .../testdata/repository/frobnitz-1.2.3.tgz | Bin 0 -> 1185 bytes .../testdata/repository/sprocket-1.2.0.tgz | Bin 0 -> 884 bytes 10 files changed, 329 insertions(+), 102 deletions(-) delete mode 100644 pkg/repo/local_test.go create mode 100644 pkg/repo/repo_test.go create mode 100644 pkg/repo/testdata/repositories.yaml create mode 100644 pkg/repo/testdata/repository/frobnitz-1.2.3.tgz create mode 100644 pkg/repo/testdata/repository/sprocket-1.2.0.tgz diff --git a/cmd/helm/repo.go b/cmd/helm/repo.go index d2c24e38e..71326f549 100644 --- a/cmd/helm/repo.go +++ b/cmd/helm/repo.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io/ioutil" + "path/filepath" "github.com/gosuri/uitable" "github.com/kubernetes/helm/pkg/repo" @@ -15,6 +16,7 @@ func init() { repoCmd.AddCommand(repoAddCmd) repoCmd.AddCommand(repoListCmd) repoCmd.AddCommand(repoRemoveCmd) + repoCmd.AddCommand(repoIndexCmd) RootCommand.AddCommand(repoCmd) } @@ -41,6 +43,12 @@ var repoRemoveCmd = &cobra.Command{ RunE: runRepoRemove, } +var repoIndexCmd = &cobra.Command{ + Use: "index [flags] [DIR]", + Short: "generate an index file for a chart repository given a directory", + RunE: runRepoIndex, +} + func runRepoAdd(cmd *cobra.Command, args []string) error { if err := checkArgsLength(2, len(args), "name for the chart repository", "the url of the chart repository"); err != nil { return err @@ -87,6 +95,35 @@ func runRepoRemove(cmd *cobra.Command, args []string) error { return nil } +func runRepoIndex(cmd *cobra.Command, args []string) error { + if err := checkArgsLength(1, len(args), "path to a directory"); err != nil { + return err + } + + path, err := filepath.Abs(args[0]) + if err != nil { + return err + } + + if err := index(path); err != nil { + return err + } + + return nil +} + +func index(dir string) error { + chartRepo, err := repo.LoadChartRepository(dir) + if err != nil { + return err + } + + if err := chartRepo.Index(); err != nil { + return err + } + return nil +} + func removeRepoLine(name string) error { r, err := repo.LoadRepositoriesFile(repositoriesFile()) if err != nil { diff --git a/cmd/helm/search.go b/cmd/helm/search.go index 579773413..706aa370f 100644 --- a/cmd/helm/search.go +++ b/cmd/helm/search.go @@ -49,7 +49,7 @@ func searchChartRefsForPattern(search string, chartRefs map[string]*repo.ChartRe matches = append(matches, k) continue } - for _, keyword := range c.Keywords { + for _, keyword := range c.Chartfile.Keywords { if strings.Contains(keyword, search) { matches = append(matches, k) } diff --git a/pkg/repo/local.go b/pkg/repo/local.go index 30fdab42d..febd1026a 100644 --- a/pkg/repo/local.go +++ b/pkg/repo/local.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/kubernetes/helm/pkg/chart" - "gopkg.in/yaml.v2" ) var localRepoPath string @@ -55,22 +54,6 @@ func AddChartToLocalRepo(ch *chart.Chart, path string) error { return nil } -// LoadIndexFile takes a file at the given path and returns an IndexFile object -func LoadIndexFile(path string) (*IndexFile, error) { - b, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - //TODO: change variable name - y is not helpful :P - var y IndexFile - err = yaml.Unmarshal(b, &y) - if err != nil { - return nil, err - } - return &y, nil -} - // Reindex adds an entry to the index file at the given path func Reindex(ch *chart.Chart, path string) error { name := ch.Chartfile().Name + "-" + ch.Chartfile().Version @@ -88,7 +71,7 @@ func Reindex(ch *chart.Chart, path string) error { if !found { url := "localhost:8879/charts/" + name + ".tgz" - out, err := y.insertChartEntry(name, url) + out, err := y.addEntry(name, url) if err != nil { return err } @@ -97,29 +80,3 @@ func Reindex(ch *chart.Chart, path string) error { } return nil } - -// UnmarshalYAML unmarshals the index file -func (i *IndexFile) UnmarshalYAML(unmarshal func(interface{}) error) error { - var refs map[string]*ChartRef - if err := unmarshal(&refs); err != nil { - if _, ok := err.(*yaml.TypeError); !ok { - return err - } - } - i.Entries = refs - return nil -} - -func (i *IndexFile) insertChartEntry(name string, url string) ([]byte, error) { - if i.Entries == nil { - i.Entries = make(map[string]*ChartRef) - } - entry := ChartRef{Name: name, URL: url} - i.Entries[name] = &entry - out, err := yaml.Marshal(&i.Entries) - if err != nil { - return nil, err - } - - return out, nil -} diff --git a/pkg/repo/local_test.go b/pkg/repo/local_test.go deleted file mode 100644 index ac83f5cd4..000000000 --- a/pkg/repo/local_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package repo - -import ( - "testing" -) - -const testfile = "testdata/local-index.yaml" - -func TestLoadIndexFile(t *testing.T) { - cf, err := LoadIndexFile(testfile) - if err != nil { - t.Errorf("Failed to load index file: %s", err) - } - if len(cf.Entries) != 2 { - t.Errorf("Expected 2 entries in the index file, but got %d", len(cf.Entries)) - } - nginx := false - alpine := false - for k, e := range cf.Entries { - if k == "nginx-0.1.0" { - if e.Name == "nginx" { - if len(e.Keywords) == 3 { - nginx = true - } - } - } - if k == "alpine-1.0.0" { - if e.Name == "alpine" { - if len(e.Keywords) == 4 { - alpine = true - } - } - } - } - if !nginx { - t.Errorf("nginx entry was not decoded properly") - } - if !alpine { - t.Errorf("alpine entry was not decoded properly") - } -} diff --git a/pkg/repo/repo.go b/pkg/repo/repo.go index 13138c10d..91af1c1e3 100644 --- a/pkg/repo/repo.go +++ b/pkg/repo/repo.go @@ -1,12 +1,41 @@ package repo import ( + "errors" "io/ioutil" + "os" + "path/filepath" + "strings" + "github.com/kubernetes/helm/pkg/chart" "gopkg.in/yaml.v2" ) -// RepoFile represents the .repositories file in $HELM_HOME +var indexPath = "index.yaml" + +// ChartRepository represents a chart repository +type ChartRepository struct { + RootPath string + URL string // URL of repository + ChartPaths []string + IndexFile *IndexFile +} + +// IndexFile represents the index file in a chart repository +type IndexFile struct { + Entries map[string]*ChartRef +} + +// ChartRef represents a chart entry in the IndexFile +type ChartRef struct { + Name string `yaml:"name"` + URL string `yaml:"url"` + Created string `yaml:"created,omitempty"` + Removed bool `yaml:"removed,omitempty"` + Chartfile chart.Chartfile `yaml:"chartfile"` +} + +// RepoFile represents the repositories.yaml file in $HELM_HOME type RepoFile struct { Repositories map[string]string } @@ -38,3 +67,127 @@ func (rf *RepoFile) UnmarshalYAML(unmarshal func(interface{}) error) error { rf.Repositories = repos return nil } + +// LoadChartRepository takes in a path to a local chart repository +// which contains packaged charts and an index.yaml file +// +// This function evaluates the contents of the directory and +// returns a ChartRepository +func LoadChartRepository(dir string) (*ChartRepository, error) { + dirInfo, err := os.Stat(dir) + if err != nil { + return nil, err + } + + if !dirInfo.IsDir() { + return nil, errors.New(dir + "is not a directory") + } + + r := &ChartRepository{RootPath: dir} + + filepath.Walk(dir, func(path string, f os.FileInfo, err error) error { + if !f.IsDir() { + if strings.Contains(f.Name(), "index.yaml") { + i, err := LoadIndexFile(path) + if err != nil { + return nil + } + r.IndexFile = i + } else { + // TODO: check for tgz extension + r.ChartPaths = append(r.ChartPaths, path) + } + } + return nil + }) + + return r, nil +} + +// UnmarshalYAML unmarshals the index file +func (i *IndexFile) UnmarshalYAML(unmarshal func(interface{}) error) error { + var refs map[string]*ChartRef + if err := unmarshal(&refs); err != nil { + if _, ok := err.(*yaml.TypeError); !ok { + return err + } + } + i.Entries = refs + return nil +} + +func (i *IndexFile) addEntry(name string, url string) ([]byte, error) { + if i.Entries == nil { + i.Entries = make(map[string]*ChartRef) + } + entry := ChartRef{Name: name, URL: url} + i.Entries[name] = &entry + out, err := yaml.Marshal(&i.Entries) + if err != nil { + return nil, err + } + + return out, nil +} + +// LoadIndexFile takes a file at the given path and returns an IndexFile object +func LoadIndexFile(path string) (*IndexFile, error) { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + var indexfile IndexFile + err = yaml.Unmarshal(b, &indexfile) + if err != nil { + return nil, err + } + + return &indexfile, nil +} + +func (r *ChartRepository) Index() error { + if r.IndexFile == nil { + r.IndexFile = &IndexFile{Entries: make(map[string]*ChartRef)} + } + + for _, path := range r.ChartPaths { + ch, err := chart.Load(path) + if err != nil { + return err + } + chartfile := ch.Chartfile() + + key := chartfile.Name + "-" + chartfile.Version + if r.IndexFile.Entries == nil { + r.IndexFile.Entries = make(map[string]*ChartRef) + } + + entry := &ChartRef{Chartfile: *chartfile, Name: chartfile.Name, URL: "", Created: "", Removed: false} + + //TODO: generate hash of contents of chart and add to the entry + //TODO: Set created timestamp + + r.IndexFile.Entries[key] = entry + + } + + if err := r.saveIndexFile(); err != nil { + return err + } + + return nil +} + +func (r *ChartRepository) saveIndexFile() error { + index, err := yaml.Marshal(&r.IndexFile.Entries) + if err != nil { + return err + } + + if err = ioutil.WriteFile(filepath.Join(r.RootPath, indexPath), index, 0644); err != nil { + return err + } + + return nil +} diff --git a/pkg/repo/repo_test.go b/pkg/repo/repo_test.go new file mode 100644 index 000000000..d677484ed --- /dev/null +++ b/pkg/repo/repo_test.go @@ -0,0 +1,114 @@ +package repo + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +const testfile = "testdata/local-index.yaml" +const testRepositoriesFile = "testdata/repositories.yaml" +const testRepository = "testdata/repository" + +func TestLoadIndexFile(t *testing.T) { + cf, err := LoadIndexFile(testfile) + if err != nil { + t.Errorf("Failed to load index file: %s", err) + } + if len(cf.Entries) != 2 { + t.Errorf("Expected 2 entries in the index file, but got %d", len(cf.Entries)) + } + nginx := false + alpine := false + for k, e := range cf.Entries { + if k == "nginx-0.1.0" { + if e.Name == "nginx" { + if len(e.Chartfile.Keywords) == 3 { + nginx = true + } + } + } + if k == "alpine-1.0.0" { + if e.Name == "alpine" { + if len(e.Chartfile.Keywords) == 4 { + alpine = true + } + } + } + } + if !nginx { + t.Errorf("nginx entry was not decoded properly") + } + if !alpine { + t.Errorf("alpine entry was not decoded properly") + } +} + +func TestLoadRepositoriesFile(t *testing.T) { + rf, err := LoadRepositoriesFile(testRepositoriesFile) + if err != nil { + t.Errorf(testRepositoriesFile + " could not be loaded: " + err.Error()) + } + expected := map[string]string{"best-charts-ever": "http://best-charts-ever.com", + "okay-charts": "http://okay-charts.org", "example123": "http://examplecharts.net/charts/123"} + + numOfRepositories := len(rf.Repositories) + expectedNumOfRepositories := 3 + if numOfRepositories != expectedNumOfRepositories { + t.Errorf("Expected %v repositories but only got %v", expectedNumOfRepositories, numOfRepositories) + } + + for expectedRepo, expectedURL := range expected { + actual, ok := rf.Repositories[expectedRepo] + if !ok { + t.Errorf("Expected repository: %v but was not found", expectedRepo) + } + + if expectedURL != actual { + t.Errorf("Expected url %s for the %s repository but got %s ", expectedURL, expectedRepo, actual) + } + } +} + +func TestLoadChartRepository(t *testing.T) { + cr, err := LoadChartRepository(testRepository) + if err != nil { + t.Errorf("Problem loading chart repository from %s: %v", testRepository, err) + } + + paths := []string{filepath.Join(testRepository, "frobnitz-1.2.3.tgz"), filepath.Join(testRepository, "sprocket-1.2.0.tgz")} + + if cr.RootPath != testRepository { + t.Errorf("Expected %s as RootPath but got %s", testRepository, cr.RootPath) + } + + if !reflect.DeepEqual(cr.ChartPaths, paths) { + t.Errorf("Expected %#v but got %#v\n", paths, cr.ChartPaths) + } +} + +func TestIndex(t *testing.T) { + cr, err := LoadChartRepository(testRepository) + if err != nil { + t.Errorf("Problem loading chart repository from %s: %v", testRepository, err) + } + + err = cr.Index() + if err != nil { + t.Errorf("Error performing index: %v\n", err) + } + + tempIndexPath := filepath.Join(testRepository, indexPath) + actual, err := LoadIndexFile(tempIndexPath) + if err != nil { + t.Errorf("Error loading index file %v", err) + } + + numEntries := len(actual.Entries) + if numEntries != 2 { + t.Errorf("Expected 2 charts to be listed in index file but got %v", numEntries) + } + + os.Remove(tempIndexPath) // clean up +} diff --git a/pkg/repo/testdata/local-index.yaml b/pkg/repo/testdata/local-index.yaml index 9d8d0e5e4..3db03faa4 100644 --- a/pkg/repo/testdata/local-index.yaml +++ b/pkg/repo/testdata/local-index.yaml @@ -1,22 +1,26 @@ nginx-0.1.0: url: http://storage.googleapis.com/kubernetes-charts/nginx-0.1.0.tgz name: nginx - description: string - version: 0.1.0 - home: https://github.com/something - keywords: - - popular - - web server - - proxy + chartfile: + name: nginx + description: string + version: 0.1.0 + home: https://github.com/something + keywords: + - popular + - web server + - proxy alpine-1.0.0: url: http://storage.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz name: alpine - description: string - version: 1.0.0 - home: https://github.com/something - keywords: - - linux - - alpine - - small - - sumtin + chartfile: + name: alpine + description: string + version: 1.0.0 + home: https://github.com/something + keywords: + - linux + - alpine + - small + - sumtin diff --git a/pkg/repo/testdata/repositories.yaml b/pkg/repo/testdata/repositories.yaml new file mode 100644 index 000000000..3fb55b060 --- /dev/null +++ b/pkg/repo/testdata/repositories.yaml @@ -0,0 +1,3 @@ +best-charts-ever: http://best-charts-ever.com +okay-charts: http://okay-charts.org +example123: http://examplecharts.net/charts/123 diff --git a/pkg/repo/testdata/repository/frobnitz-1.2.3.tgz b/pkg/repo/testdata/repository/frobnitz-1.2.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..0fe27e994ac89f5de5af4ed0259a114c0b557f28 GIT binary patch literal 1185 zcmV;S1YY|eiwG0|32ul0|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PLN;ZzIJU$LFs26{8X$0u3#6G|*AeKtV-K4Fw$)B^?s{0Z2S+Z@f6zP6UoalJ7_NXm)oV z&#e8-GdH8`SKH5ZYPykm%uWx-37mGbNu}eoTjSF4f^j*9xnA3C1#aN9eCD{W7qpmY zp5i@C7#2q6bSwY1ox?A3Se_Fm@wcs81DPA^Kqj$O?CT}=;kb^|3W7QD_ow1dp(p>q z^&BQv@*`gI_;;Om?%oq^D=1H%XUu=8^I)*~@aorpFqTBUfr?|5YMskCn%hQFIasmo z$#{NW9~^1W-qAfBD!a&g_D*C5!>-lSiM>1QsytOj6*hS%cDB1?=PJ`ST`{3Fa&%Ti zM&}21J$~!qyYwFkfb-7K^fz96j`^NDnw!=eFN+snxPR7n1poj500013Att-RKBV$$ zb$+cXuN)}>0000000000aDI^?W1p@u_A6QW_y5EF|L=70BiVOk-;#Yr_66BzWFM35 zlRYB4Lw1Ag6|(JrqAA&NfxW}k#n{& zx>!tKwfkDzU752&5A&YtFdO!YVY{iADKTtl$KGS`B zpu`=j6x&KBjGR+Lrt+dgE};%-d%huxEZ4nVW$3V<>uy@Ab#~RkzRpAP+avEtryb#1 zp5?Pp6}>#l%;?hgASwj?lhp?wTWs9#q@8*Ck8FuSIJy4AX?b3y{|KP}SOrJ*A5Z_^ zdGOP#53gST4f+oNE)39r0000000405f&K#k000000002E_>}sO;TmH<%+Y^*L-r-v z=VYIdeN<^b-XptJX+PE~?Z+=wJKsb90RR9100000;53=5?-1go3Xeq^kFu4SDv$X( zkNP7ElpdwiSnE+9P1GLc$s)Z+VvY=EH!_sN&zp3_Atg}R=hV$Dx7vcDV%0-m+ z^$k%dB@E30MrRvwStn{;mvy0$m~1+)N*wRrNxdXv^z;weqieL0McXwbSY4ja#&VIscJ)cLDc zVQyr3R8em|NM&qo0PLGhZreB%fSCZZAu!2cW)8q{mVqTomhFLo022%bGqY$nP1{A& z1ufCGU`Z52Dyf^My+D`UblH9PUH1mPKyT1RdxJjtM~;&?DNx&K;P)X9lt|`L^7}Cb z>*+L8eqS20L^$_&vtEam^JZhza@QNRBZ%WRordRlPBHE{yx|g7U*<(k$h8p}T*~+A zncT>vy(LWSZ(D!&L}sjkh$BlMr;A*JJDfK>Z;t(IXYKF0jtlc^&6-2lNI z`2P-TT3&JT3n9_(t%rO2hnu^f35i3$C!Z6Rb%2BAUO}^K%OAuJTnN zZJqgcH#EJxZTTv;`*~Yti8NB%Fgv!>e%H=qs%+>nfiPltrbDB$fnECV)O{EJ86n86 zCpVqypFfkD>kM<#PUR2w+pqU;`l=8FK@bE%5CmBvCfy(}pxqpwZ;#uJb1i}(2!bF8 zf*=TjAh#D05b|n^kWV1R@BfqM|1WUx0pvZ%JCL^^Z$MsyyaYJ{IRyC^WEW%yAP9mW2!bHU`q5LdY_Xm(Y0I|d?Aa3$r;#k)I49L%r=IP#J4)GYkrA!( z%$F^~n2bdjwOFFsDj57Vt057E@_Vp^N+kOy(sKF_#cS-BPCw{L_8$zg11VwxlM@3X zGu?tIfePU1yvoWFohs{Os-1-9De23>vC0CNcLcLTm9&^+xfUmZ)P5GGX4rMm3pIm3 zQR=9W(ZZ$9RlGlcb^QnD9vn~Ve_Y2a>OXkBf%PA&VAdVzb?!rF-Hn!hNT<$8r!Gs! zUQExrHoesQ>D*|cmjxHSA{yy9V(FD}P3NTpdeb&pC1v~@3HL9I)Qex^o9q8-b>27= z|5*RGd@kC*JhDu>nD2t?@;`h5oU?!3E9yTn|1TfNzmZ}mC)Qm^vFsTOWJlzYVHHU0 zQBXNOU8`)kew6WlB%)ku%c#X~0I!RG=R*9~YHl6le--SoeKs;Z>x7YHz8db+baBuL zyLl#xy`V*Du?1s$tWs>kSdlC$ls?kZFNPnNqr)r`CdHO Date: Wed, 18 May 2016 15:00:35 -0400 Subject: [PATCH 2/5] feat(helm): add chart url to index file entries --- cmd/helm/repo.go | 10 +++--- pkg/repo/index.go | 55 +++++++++++++++++++++++++++--- pkg/repo/repo.go | 79 ++++++------------------------------------- pkg/repo/repo_test.go | 9 +++-- 4 files changed, 74 insertions(+), 79 deletions(-) diff --git a/cmd/helm/repo.go b/cmd/helm/repo.go index 71326f549..cb09b0046 100644 --- a/cmd/helm/repo.go +++ b/cmd/helm/repo.go @@ -44,7 +44,7 @@ var repoRemoveCmd = &cobra.Command{ } var repoIndexCmd = &cobra.Command{ - Use: "index [flags] [DIR]", + Use: "index [flags] [DIR] [REPO_URL]", Short: "generate an index file for a chart repository given a directory", RunE: runRepoIndex, } @@ -96,7 +96,7 @@ func runRepoRemove(cmd *cobra.Command, args []string) error { } func runRepoIndex(cmd *cobra.Command, args []string) error { - if err := checkArgsLength(1, len(args), "path to a directory"); err != nil { + if err := checkArgsLength(2, len(args), "path to a directory", "url of chart repository"); err != nil { return err } @@ -105,15 +105,15 @@ func runRepoIndex(cmd *cobra.Command, args []string) error { return err } - if err := index(path); err != nil { + if err := index(path, args[1]); err != nil { return err } return nil } -func index(dir string) error { - chartRepo, err := repo.LoadChartRepository(dir) +func index(dir, url string) error { + chartRepo, err := repo.LoadChartRepository(dir, url) if err != nil { return err } diff --git a/pkg/repo/index.go b/pkg/repo/index.go index 33f9c1dfe..feee9cdaa 100644 --- a/pkg/repo/index.go +++ b/pkg/repo/index.go @@ -6,8 +6,12 @@ import ( "strings" "gopkg.in/yaml.v2" + + "github.com/kubernetes/helm/pkg/chart" ) +var indexPath = "index.yaml" + // IndexFile represents the index file in a chart repository type IndexFile struct { Entries map[string]*ChartRef @@ -15,10 +19,11 @@ type IndexFile struct { // ChartRef represents a chart entry in the IndexFile type ChartRef struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - Keywords []string `yaml:"keywords"` - Removed bool `yaml:"removed,omitempty"` + Name string `yaml:"name"` + URL string `yaml:"url"` + Created string `yaml:"created,omitempty"` + Removed bool `yaml:"removed,omitempty"` + Chartfile chart.Chartfile `yaml:"chartfile"` } // DownloadIndexFile uses @@ -49,3 +54,45 @@ func DownloadIndexFile(repoName, url, indexFileName string) error { return nil } + +// UnmarshalYAML unmarshals the index file +func (i *IndexFile) UnmarshalYAML(unmarshal func(interface{}) error) error { + var refs map[string]*ChartRef + if err := unmarshal(&refs); err != nil { + if _, ok := err.(*yaml.TypeError); !ok { + return err + } + } + i.Entries = refs + return nil +} + +func (i *IndexFile) addEntry(name string, url string) ([]byte, error) { + if i.Entries == nil { + i.Entries = make(map[string]*ChartRef) + } + entry := ChartRef{Name: name, URL: url} + i.Entries[name] = &entry + out, err := yaml.Marshal(&i.Entries) + if err != nil { + return nil, err + } + + return out, nil +} + +// LoadIndexFile takes a file at the given path and returns an IndexFile object +func LoadIndexFile(path string) (*IndexFile, error) { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + var indexfile IndexFile + err = yaml.Unmarshal(b, &indexfile) + if err != nil { + return nil, err + } + + return &indexfile, nil +} diff --git a/pkg/repo/repo.go b/pkg/repo/repo.go index 91af1c1e3..2ad6e3a94 100644 --- a/pkg/repo/repo.go +++ b/pkg/repo/repo.go @@ -7,11 +7,10 @@ import ( "path/filepath" "strings" - "github.com/kubernetes/helm/pkg/chart" "gopkg.in/yaml.v2" -) -var indexPath = "index.yaml" + "github.com/kubernetes/helm/pkg/chart" +) // ChartRepository represents a chart repository type ChartRepository struct { @@ -21,20 +20,6 @@ type ChartRepository struct { IndexFile *IndexFile } -// IndexFile represents the index file in a chart repository -type IndexFile struct { - Entries map[string]*ChartRef -} - -// ChartRef represents a chart entry in the IndexFile -type ChartRef struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - Created string `yaml:"created,omitempty"` - Removed bool `yaml:"removed,omitempty"` - Chartfile chart.Chartfile `yaml:"chartfile"` -} - // RepoFile represents the repositories.yaml file in $HELM_HOME type RepoFile struct { Repositories map[string]string @@ -73,7 +58,7 @@ func (rf *RepoFile) UnmarshalYAML(unmarshal func(interface{}) error) error { // // This function evaluates the contents of the directory and // returns a ChartRepository -func LoadChartRepository(dir string) (*ChartRepository, error) { +func LoadChartRepository(dir, url string) (*ChartRepository, error) { dirInfo, err := os.Stat(dir) if err != nil { return nil, err @@ -83,7 +68,7 @@ func LoadChartRepository(dir string) (*ChartRepository, error) { return nil, errors.New(dir + "is not a directory") } - r := &ChartRepository{RootPath: dir} + r := &ChartRepository{RootPath: dir, URL: url} filepath.Walk(dir, func(path string, f os.FileInfo, err error) error { if !f.IsDir() { @@ -104,46 +89,17 @@ func LoadChartRepository(dir string) (*ChartRepository, error) { return r, nil } -// UnmarshalYAML unmarshals the index file -func (i *IndexFile) UnmarshalYAML(unmarshal func(interface{}) error) error { - var refs map[string]*ChartRef - if err := unmarshal(&refs); err != nil { - if _, ok := err.(*yaml.TypeError); !ok { - return err - } - } - i.Entries = refs - return nil -} - -func (i *IndexFile) addEntry(name string, url string) ([]byte, error) { - if i.Entries == nil { - i.Entries = make(map[string]*ChartRef) - } - entry := ChartRef{Name: name, URL: url} - i.Entries[name] = &entry - out, err := yaml.Marshal(&i.Entries) - if err != nil { - return nil, err - } - - return out, nil -} - -// LoadIndexFile takes a file at the given path and returns an IndexFile object -func LoadIndexFile(path string) (*IndexFile, error) { - b, err := ioutil.ReadFile(path) +func (r *ChartRepository) saveIndexFile() error { + index, err := yaml.Marshal(&r.IndexFile.Entries) if err != nil { - return nil, err + return err } - var indexfile IndexFile - err = yaml.Unmarshal(b, &indexfile) - if err != nil { - return nil, err + if err = ioutil.WriteFile(filepath.Join(r.RootPath, indexPath), index, 0644); err != nil { + return err } - return &indexfile, nil + return nil } func (r *ChartRepository) Index() error { @@ -163,7 +119,7 @@ func (r *ChartRepository) Index() error { r.IndexFile.Entries = make(map[string]*ChartRef) } - entry := &ChartRef{Chartfile: *chartfile, Name: chartfile.Name, URL: "", Created: "", Removed: false} + entry := &ChartRef{Chartfile: *chartfile, Name: chartfile.Name, URL: r.URL, Created: "", Removed: false} //TODO: generate hash of contents of chart and add to the entry //TODO: Set created timestamp @@ -178,16 +134,3 @@ func (r *ChartRepository) Index() error { return nil } - -func (r *ChartRepository) saveIndexFile() error { - index, err := yaml.Marshal(&r.IndexFile.Entries) - if err != nil { - return err - } - - if err = ioutil.WriteFile(filepath.Join(r.RootPath, indexPath), index, 0644); err != nil { - return err - } - - return nil -} diff --git a/pkg/repo/repo_test.go b/pkg/repo/repo_test.go index d677484ed..2ab4dcf65 100644 --- a/pkg/repo/repo_test.go +++ b/pkg/repo/repo_test.go @@ -10,6 +10,7 @@ import ( const testfile = "testdata/local-index.yaml" const testRepositoriesFile = "testdata/repositories.yaml" const testRepository = "testdata/repository" +const testURL = "http://example-charts.com" func TestLoadIndexFile(t *testing.T) { cf, err := LoadIndexFile(testfile) @@ -72,7 +73,7 @@ func TestLoadRepositoriesFile(t *testing.T) { } func TestLoadChartRepository(t *testing.T) { - cr, err := LoadChartRepository(testRepository) + cr, err := LoadChartRepository(testRepository, testURL) if err != nil { t.Errorf("Problem loading chart repository from %s: %v", testRepository, err) } @@ -86,10 +87,14 @@ func TestLoadChartRepository(t *testing.T) { if !reflect.DeepEqual(cr.ChartPaths, paths) { t.Errorf("Expected %#v but got %#v\n", paths, cr.ChartPaths) } + + if cr.URL != testURL { + t.Errorf("Expected url for chart repository to be %s but got %s", testURL, cr.URL) + } } func TestIndex(t *testing.T) { - cr, err := LoadChartRepository(testRepository) + cr, err := LoadChartRepository(testRepository, testURL) if err != nil { t.Errorf("Problem loading chart repository from %s: %v", testRepository, err) } From 7bd739c27ed7b5ea95ef280f88d57abffb938bdd Mon Sep 17 00:00:00 2001 From: Michelle Noorali Date: Fri, 20 May 2016 10:59:22 -0400 Subject: [PATCH 3/5] feat(helm): populate Digest & Created in index func --- pkg/repo/index.go | 1 + pkg/repo/index_test.go | 1 + pkg/repo/repo.go | 28 ++++++++++++++++++++++++---- pkg/repo/repo_test.go | 21 ++++++++++++++++++++- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/pkg/repo/index.go b/pkg/repo/index.go index feee9cdaa..6e2c1920d 100644 --- a/pkg/repo/index.go +++ b/pkg/repo/index.go @@ -23,6 +23,7 @@ type ChartRef struct { URL string `yaml:"url"` Created string `yaml:"created,omitempty"` Removed bool `yaml:"removed,omitempty"` + Digest string `yaml:"digest,omitempty"` Chartfile chart.Chartfile `yaml:"chartfile"` } diff --git a/pkg/repo/index_test.go b/pkg/repo/index_test.go index 41b04d2c4..0b5590c96 100644 --- a/pkg/repo/index_test.go +++ b/pkg/repo/index_test.go @@ -52,5 +52,6 @@ func TestDownloadIndexFile(t *testing.T) { t.Errorf("Expected 2 entries in index file but got %v", numEntries) } os.Remove(path) + os.Remove(dirName) } diff --git a/pkg/repo/repo.go b/pkg/repo/repo.go index 2ad6e3a94..7f2ce4d01 100644 --- a/pkg/repo/repo.go +++ b/pkg/repo/repo.go @@ -1,11 +1,14 @@ package repo import ( + "crypto/sha1" "errors" + "fmt" "io/ioutil" "os" "path/filepath" "strings" + "time" "gopkg.in/yaml.v2" @@ -113,16 +116,17 @@ func (r *ChartRepository) Index() error { return err } chartfile := ch.Chartfile() + hash, err := generateDigest(path) + if err != nil { + return err + } key := chartfile.Name + "-" + chartfile.Version if r.IndexFile.Entries == nil { r.IndexFile.Entries = make(map[string]*ChartRef) } - entry := &ChartRef{Chartfile: *chartfile, Name: chartfile.Name, URL: r.URL, Created: "", Removed: false} - - //TODO: generate hash of contents of chart and add to the entry - //TODO: Set created timestamp + entry := &ChartRef{Chartfile: *chartfile, Name: chartfile.Name, URL: r.URL, Created: time.Now().UTC().String(), Digest: hash, Removed: false} r.IndexFile.Entries[key] = entry @@ -134,3 +138,19 @@ func (r *ChartRepository) Index() error { return nil } + +func generateDigest(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + + b, err := ioutil.ReadAll(f) + if err != nil { + return "", err + } + + result := sha1.Sum(b) + + return fmt.Sprintf("%x", result), nil +} diff --git a/pkg/repo/repo_test.go b/pkg/repo/repo_test.go index 2ab4dcf65..f765091fd 100644 --- a/pkg/repo/repo_test.go +++ b/pkg/repo/repo_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "reflect" "testing" + "time" ) const testfile = "testdata/local-index.yaml" @@ -110,10 +111,28 @@ func TestIndex(t *testing.T) { t.Errorf("Error loading index file %v", err) } - numEntries := len(actual.Entries) + entries := actual.Entries + numEntries := len(entries) if numEntries != 2 { t.Errorf("Expected 2 charts to be listed in index file but got %v", numEntries) } + var empty time.Time + for chartName, details := range entries { + if details == nil { + t.Errorf("Chart Entry is not filled out for %s", chartName) + } + + if details.Created == empty.String() { + t.Errorf("Created timestamp under %s chart entry is nil", chartName) + } + + if details.Digest == "" { + t.Errorf("Digest was not set for %s", chartName) + } + } + + //TODO: test update case + os.Remove(tempIndexPath) // clean up } From 2234dc89274a0be1588b983b7450b8c3ad55e065 Mon Sep 17 00:00:00 2001 From: Michelle Noorali Date: Fri, 20 May 2016 11:21:15 -0400 Subject: [PATCH 4/5] feat(helm): handle update case on index function --- pkg/repo/repo.go | 11 ++++++++++- pkg/repo/repo_test.go | 21 +++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/pkg/repo/repo.go b/pkg/repo/repo.go index 7f2ce4d01..c8004905d 100644 --- a/pkg/repo/repo.go +++ b/pkg/repo/repo.go @@ -115,6 +115,7 @@ func (r *ChartRepository) Index() error { if err != nil { return err } + chartfile := ch.Chartfile() hash, err := generateDigest(path) if err != nil { @@ -126,7 +127,15 @@ func (r *ChartRepository) Index() error { r.IndexFile.Entries = make(map[string]*ChartRef) } - entry := &ChartRef{Chartfile: *chartfile, Name: chartfile.Name, URL: r.URL, Created: time.Now().UTC().String(), Digest: hash, Removed: false} + ref, ok := r.IndexFile.Entries[key] + var created string + if ok && ref.Created != "" { + created = ref.Created + } else { + created = time.Now().UTC().String() + } + + entry := &ChartRef{Chartfile: *chartfile, Name: chartfile.Name, URL: r.URL, Created: created, Digest: hash, Removed: false} r.IndexFile.Entries[key] = entry diff --git a/pkg/repo/repo_test.go b/pkg/repo/repo_test.go index f765091fd..1ad181813 100644 --- a/pkg/repo/repo_test.go +++ b/pkg/repo/repo_test.go @@ -107,6 +107,7 @@ func TestIndex(t *testing.T) { tempIndexPath := filepath.Join(testRepository, indexPath) actual, err := LoadIndexFile(tempIndexPath) + defer os.Remove(tempIndexPath) // clean up if err != nil { t.Errorf("Error loading index file %v", err) } @@ -117,6 +118,7 @@ func TestIndex(t *testing.T) { t.Errorf("Expected 2 charts to be listed in index file but got %v", numEntries) } + timestamps := make(map[string]string) var empty time.Time for chartName, details := range entries { if details == nil { @@ -126,13 +128,28 @@ func TestIndex(t *testing.T) { if details.Created == empty.String() { t.Errorf("Created timestamp under %s chart entry is nil", chartName) } + timestamps[chartName] = details.Created if details.Digest == "" { t.Errorf("Digest was not set for %s", chartName) } } - //TODO: test update case + if err = cr.Index(); err != nil { + t.Errorf("Error performing index the second time: %v\n", err) + } + second, err := LoadIndexFile(tempIndexPath) + if err != nil { + t.Errorf("Error loading index file second time: %#v\n", err) + } - os.Remove(tempIndexPath) // clean up + for chart, created := range timestamps { + v, ok := second.Entries[chart] + if !ok { + t.Errorf("Expected %s chart entry in index file but did not find it", chart) + } + if v.Created != created { + t.Errorf("Expected Created timestamp to be %s, but got %s for chart %s", created, v.Created, chart) + } + } } From 0783fee7b7d8f6afde9baa511319ecf9d72b9de3 Mon Sep 17 00:00:00 2001 From: Michelle Noorali Date: Fri, 20 May 2016 14:28:58 -0400 Subject: [PATCH 5/5] ref(helm): refactor test index files for search --- cmd/helm/search_test.go | 4 +- cmd/helm/testdata/foobar-index.yaml | 30 ++++++---- cmd/helm/testdata/local-index.yaml | 36 +++++++----- pkg/repo/index_test.go | 36 ++++++++++++ pkg/repo/repo_test.go | 87 +++++++++-------------------- 5 files changed, 103 insertions(+), 90 deletions(-) diff --git a/cmd/helm/search_test.go b/cmd/helm/search_test.go index 7001cdec3..5a54bc70d 100644 --- a/cmd/helm/search_test.go +++ b/cmd/helm/search_test.go @@ -1,9 +1,9 @@ package main import ( - "github.com/kubernetes/helm/pkg/repo" - "testing" + + "github.com/kubernetes/helm/pkg/repo" ) const testDir = "testdata/" diff --git a/cmd/helm/testdata/foobar-index.yaml b/cmd/helm/testdata/foobar-index.yaml index ef28fdc48..c23e5ea0c 100644 --- a/cmd/helm/testdata/foobar-index.yaml +++ b/cmd/helm/testdata/foobar-index.yaml @@ -1,18 +1,24 @@ foobar-0.1.0: url: http://storage.googleapis.com/kubernetes-charts/nginx-0.1.0.tgz name: foobar - description: string - version: 0.1.0 - home: https://github.com/foo - keywords: - - dummy - - hokey + removed: false + chartfile: + name: foobar + description: string + version: 0.1.0 + home: https://github.com/foo + keywords: + - dummy + - hokey oddness-1.2.3: url: http://storage.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz name: oddness - description: string - version: 1.2.3 - home: https://github.com/something - keywords: - - duck - - sumtin + removed: false + chartfile: + name: oddness + description: string + version: 1.2.3 + home: https://github.com/something + keywords: + - duck + - sumtin diff --git a/cmd/helm/testdata/local-index.yaml b/cmd/helm/testdata/local-index.yaml index 9d8d0e5e4..efaf5e46e 100644 --- a/cmd/helm/testdata/local-index.yaml +++ b/cmd/helm/testdata/local-index.yaml @@ -1,22 +1,28 @@ nginx-0.1.0: url: http://storage.googleapis.com/kubernetes-charts/nginx-0.1.0.tgz name: nginx - description: string - version: 0.1.0 - home: https://github.com/something - keywords: - - popular - - web server - - proxy + removed: false + chartfile: + name: nginx + description: string + version: 0.1.0 + home: https://github.com/something + keywords: + - popular + - web server + - proxy alpine-1.0.0: url: http://storage.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz name: alpine - description: string - version: 1.0.0 - home: https://github.com/something - keywords: - - linux - - alpine - - small - - sumtin + removed: false + chartfile: + name: alpine + description: string + version: 1.0.0 + home: https://github.com/something + keywords: + - linux + - alpine + - small + - sumtin diff --git a/pkg/repo/index_test.go b/pkg/repo/index_test.go index 0b5590c96..e891c8690 100644 --- a/pkg/repo/index_test.go +++ b/pkg/repo/index_test.go @@ -12,6 +12,8 @@ import ( "gopkg.in/yaml.v2" ) +const testfile = "testdata/local-index.yaml" + var ( testRepo = "test-repo" ) @@ -55,3 +57,37 @@ func TestDownloadIndexFile(t *testing.T) { os.Remove(dirName) } + +func TestLoadIndexFile(t *testing.T) { + cf, err := LoadIndexFile(testfile) + if err != nil { + t.Errorf("Failed to load index file: %s", err) + } + if len(cf.Entries) != 2 { + t.Errorf("Expected 2 entries in the index file, but got %d", len(cf.Entries)) + } + nginx := false + alpine := false + for k, e := range cf.Entries { + if k == "nginx-0.1.0" { + if e.Name == "nginx" { + if len(e.Chartfile.Keywords) == 3 { + nginx = true + } + } + } + if k == "alpine-1.0.0" { + if e.Name == "alpine" { + if len(e.Chartfile.Keywords) == 4 { + alpine = true + } + } + } + } + if !nginx { + t.Errorf("nginx entry was not decoded properly") + } + if !alpine { + t.Errorf("alpine entry was not decoded properly") + } +} diff --git a/pkg/repo/repo_test.go b/pkg/repo/repo_test.go index 1ad181813..3680691c5 100644 --- a/pkg/repo/repo_test.go +++ b/pkg/repo/repo_test.go @@ -8,71 +8,10 @@ import ( "time" ) -const testfile = "testdata/local-index.yaml" const testRepositoriesFile = "testdata/repositories.yaml" const testRepository = "testdata/repository" const testURL = "http://example-charts.com" -func TestLoadIndexFile(t *testing.T) { - cf, err := LoadIndexFile(testfile) - if err != nil { - t.Errorf("Failed to load index file: %s", err) - } - if len(cf.Entries) != 2 { - t.Errorf("Expected 2 entries in the index file, but got %d", len(cf.Entries)) - } - nginx := false - alpine := false - for k, e := range cf.Entries { - if k == "nginx-0.1.0" { - if e.Name == "nginx" { - if len(e.Chartfile.Keywords) == 3 { - nginx = true - } - } - } - if k == "alpine-1.0.0" { - if e.Name == "alpine" { - if len(e.Chartfile.Keywords) == 4 { - alpine = true - } - } - } - } - if !nginx { - t.Errorf("nginx entry was not decoded properly") - } - if !alpine { - t.Errorf("alpine entry was not decoded properly") - } -} - -func TestLoadRepositoriesFile(t *testing.T) { - rf, err := LoadRepositoriesFile(testRepositoriesFile) - if err != nil { - t.Errorf(testRepositoriesFile + " could not be loaded: " + err.Error()) - } - expected := map[string]string{"best-charts-ever": "http://best-charts-ever.com", - "okay-charts": "http://okay-charts.org", "example123": "http://examplecharts.net/charts/123"} - - numOfRepositories := len(rf.Repositories) - expectedNumOfRepositories := 3 - if numOfRepositories != expectedNumOfRepositories { - t.Errorf("Expected %v repositories but only got %v", expectedNumOfRepositories, numOfRepositories) - } - - for expectedRepo, expectedURL := range expected { - actual, ok := rf.Repositories[expectedRepo] - if !ok { - t.Errorf("Expected repository: %v but was not found", expectedRepo) - } - - if expectedURL != actual { - t.Errorf("Expected url %s for the %s repository but got %s ", expectedURL, expectedRepo, actual) - } - } -} - func TestLoadChartRepository(t *testing.T) { cr, err := LoadChartRepository(testRepository, testURL) if err != nil { @@ -153,3 +92,29 @@ func TestIndex(t *testing.T) { } } } + +func TestLoadRepositoriesFile(t *testing.T) { + rf, err := LoadRepositoriesFile(testRepositoriesFile) + if err != nil { + t.Errorf(testRepositoriesFile + " could not be loaded: " + err.Error()) + } + expected := map[string]string{"best-charts-ever": "http://best-charts-ever.com", + "okay-charts": "http://okay-charts.org", "example123": "http://examplecharts.net/charts/123"} + + numOfRepositories := len(rf.Repositories) + expectedNumOfRepositories := 3 + if numOfRepositories != expectedNumOfRepositories { + t.Errorf("Expected %v repositories but only got %v", expectedNumOfRepositories, numOfRepositories) + } + + for expectedRepo, expectedURL := range expected { + actual, ok := rf.Repositories[expectedRepo] + if !ok { + t.Errorf("Expected repository: %v but was not found", expectedRepo) + } + + if expectedURL != actual { + t.Errorf("Expected url %s for the %s repository but got %s ", expectedURL, expectedRepo, actual) + } + } +}