From aca4fb06ce910817d0778306f678456b02901ce3 Mon Sep 17 00:00:00 2001 From: Karuppiah Natarajan Date: Fri, 31 Jan 2020 20:47:36 +0530 Subject: [PATCH] feat(repo_index): generate index.json too for repo index command Signed-off-by: Karuppiah Natarajan --- cmd/helm/repo_index.go | 94 +++++++-- cmd/helm/repo_index_test.go | 318 +++++++++++++++++++++++++++-- pkg/repo/index.go | 38 +++- pkg/repo/index_test.go | 21 ++ pkg/repo/testdata/local-index.json | 69 +++++++ 5 files changed, 496 insertions(+), 44 deletions(-) create mode 100644 pkg/repo/testdata/local-index.json diff --git a/cmd/helm/repo_index.go b/cmd/helm/repo_index.go index 63afaf37b..4c716284b 100644 --- a/cmd/helm/repo_index.go +++ b/cmd/helm/repo_index.go @@ -31,18 +31,20 @@ import ( const repoIndexDesc = ` Read the current directory and generate an index file based on the charts found. -This tool is used for creating an 'index.yaml' file for a chart repository. To +This tool is used for creating an 'index.yaml' and 'index.json' files for a chart repository. To set an absolute URL to the charts, use '--url' flag. To merge the generated index with an existing index file, use the '--merge' -flag. In this case, the charts found in the current directory will be merged +flag for 'index.yaml' file and '--merge-json-index' for 'index.json' file. Only one of the +flags can be passed at once. In this case, the charts found in the current directory will be merged into the existing index, with local charts taking priority over existing charts. ` type repoIndexOptions struct { - dir string - url string - merge string + dir string + url string + merge string + mergeJSONIndex string } func newRepoIndexCmd(out io.Writer) *cobra.Command { @@ -61,41 +63,89 @@ func newRepoIndexCmd(out io.Writer) *cobra.Command { f := cmd.Flags() f.StringVar(&o.url, "url", "", "url of chart repository") - f.StringVar(&o.merge, "merge", "", "merge the generated index into the given index") + f.StringVar(&o.merge, "merge", "", "merge the generated index into the given index.yaml") + f.StringVar(&o.mergeJSONIndex, "merge-json-index", "", "merge the generated index into the given index.json") return cmd } func (i *repoIndexOptions) run(out io.Writer) error { + if i.merge != "" && i.mergeJSONIndex != "" { + return errors.New("only one of --merge and --merge-json-index can be passed") + } + path, err := filepath.Abs(i.dir) if err != nil { return err } - return index(path, i.url, i.merge) + return index(path, i.url, i.merge, i.mergeJSONIndex) } -func index(dir, url, mergeTo string) error { - out := filepath.Join(dir, "index.yaml") +func index(dir, url, mergeTo, mergeToJSON string) error { + yamlIndex := filepath.Join(dir, "index.yaml") + jsonIndex := filepath.Join(dir, "index.json") i, err := repo.IndexDirectory(dir, url) if err != nil { return err } - if mergeTo != "" { - // if index.yaml is missing then create an empty one to merge into - var i2 *repo.IndexFile - if _, err := os.Stat(mergeTo); os.IsNotExist(err) { - i2 = repo.NewIndexFile() - i2.WriteFile(mergeTo, 0644) - } else { - i2, err = repo.LoadIndexFile(mergeTo) - if err != nil { - return errors.Wrap(err, "merge failed") - } + if mergeTo != "" || mergeToJSON != "" { + if err = mergeToIndex(i, mergeTo, mergeToJSON); err != nil { + return err } - i.Merge(i2) } i.SortEntries() - return i.WriteFile(out, 0644) + err = i.WriteFile(yamlIndex, 0644) + if err != nil { + return err + } + return i.WriteJSONFile(jsonIndex, 0644) +} + +func mergeToIndex(i *repo.IndexFile, mergeTo, mergeToJSON string) error { + // if index.yaml is missing then create an empty one to merge into + var i2 *repo.IndexFile + var err error + if mergeTo != "" { + i2, err = loadIndexForMerge(mergeTo) + if err != nil { + return err + } + } else if mergeToJSON != "" { + i2, err = loadIndexJSONForMerge(mergeToJSON) + if err != nil { + return err + } + } + i.Merge(i2) + return nil +} + +func loadIndexForMerge(mergeTo string) (*repo.IndexFile, error) { + var i2 *repo.IndexFile + if _, err := os.Stat(mergeTo); os.IsNotExist(err) { + i2 = repo.NewIndexFile() + i2.WriteFile(mergeTo, 0644) + } else { + i2, err = repo.LoadIndexFile(mergeTo) + if err != nil { + return nil, errors.Wrap(err, "merge failed") + } + } + return i2, nil +} + +func loadIndexJSONForMerge(mergeToJSON string) (*repo.IndexFile, error) { + var i2 *repo.IndexFile + if _, err := os.Stat(mergeToJSON); os.IsNotExist(err) { + i2 = repo.NewIndexFile() + i2.WriteJSONFile(mergeToJSON, 0644) + } else { + i2, err = repo.LoadIndexJSONFile(mergeToJSON) + if err != nil { + return nil, errors.Wrap(err, "merge failed") + } + } + return i2, nil } diff --git a/cmd/helm/repo_index_test.go b/cmd/helm/repo_index_test.go index e04ae1b59..d0da69322 100644 --- a/cmd/helm/repo_index_test.go +++ b/cmd/helm/repo_index_test.go @@ -21,6 +21,7 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "helm.sh/helm/v3/internal/test/ensure" @@ -28,7 +29,7 @@ import ( ) func TestRepoIndexCmd(t *testing.T) { - + // t.Run("helm repo index command", func(t *testing.T) { dir := ensure.TempDir(t) comp := filepath.Join(dir, "compressedchart-0.1.0.tgz") @@ -42,6 +43,8 @@ func TestRepoIndexCmd(t *testing.T) { buf := bytes.NewBuffer(nil) c := newRepoIndexCmd(buf) + expectedNumberOfEntries := 1 + expectedNumberOfVersions := 2 if err := c.RunE(c, []string{dir}); err != nil { t.Error(err) @@ -54,13 +57,15 @@ func TestRepoIndexCmd(t *testing.T) { t.Fatal(err) } - if len(index.Entries) != 1 { - t.Errorf("expected 1 entry, got %d: %#v", len(index.Entries), index.Entries) + if len(index.Entries) != expectedNumberOfEntries { + t.Errorf("expected %d entry, got %d: %#v", + expectedNumberOfEntries, len(index.Entries), index.Entries) } vs := index.Entries["compressedchart"] - if len(vs) != 2 { - t.Errorf("expected 2 versions, got %d: %#v", len(vs), vs) + if len(vs) != expectedNumberOfVersions { + t.Errorf("expected %d versions, got %d: %#v", + expectedNumberOfVersions, len(vs), vs) } expectedVersion := "0.2.0" @@ -68,8 +73,52 @@ func TestRepoIndexCmd(t *testing.T) { t.Errorf("expected %q, got %q", expectedVersion, vs[0].Version) } - // Test with `--merge` + // Test creation of index.json + destJSONIndex := filepath.Join(dir, "index.json") + jsonIndex, err := repo.LoadIndexJSONFile(destJSONIndex) + if err != nil { + t.Fatal(err) + } + + if len(jsonIndex.Entries) != expectedNumberOfEntries { + t.Errorf("expected %d entry in json index, got %d: %#v", + expectedNumberOfEntries, len(jsonIndex.Entries), jsonIndex.Entries) + } + + versionsInJSONIndex := jsonIndex.Entries["compressedchart"] + if len(versionsInJSONIndex) != expectedNumberOfVersions { + t.Errorf("expected %d versions in json index, got %d: %#v", + expectedNumberOfVersions, len(versionsInJSONIndex), versionsInJSONIndex) + } + + expectedVersionInJSONIndex := "0.2.0" + if versionsInJSONIndex[0].Version != expectedVersionInJSONIndex { + t.Errorf("expected %q in json index, got %q", + expectedVersionInJSONIndex, versionsInJSONIndex[0].Version) + } + + // save a copy of index.json to test --merge-json-index later + indexForMergeJSON := filepath.Join(dir, "indexForMerge.json") + if err = copyFile(destJSONIndex, indexForMergeJSON); err != nil { + t.Fatal(err) + } + + // save a copy of index.yaml to test --merge now + indexForMergeYAML := filepath.Join(dir, "indexForMerge.yaml") + if err = copyFile(destIndex, indexForMergeYAML); err != nil { + t.Fatal(err) + } + + // Test with `--merge` + // cleanup index.json and index.yaml as it's not needed. it will be created + // by the command + if err := os.Remove(destJSONIndex); err != nil { + t.Fatal(err) + } + if err := os.Remove(destIndex); err != nil { + t.Fatal(err) + } // Remove first two charts. if err := os.Remove(comp); err != nil { t.Fatal(err) @@ -85,7 +134,144 @@ func TestRepoIndexCmd(t *testing.T) { t.Fatal(err) } - c.ParseFlags([]string{"--merge", destIndex}) + buf = bytes.NewBuffer(nil) + c = newRepoIndexCmd(buf) + expectedNumberOfEntries = 2 + expectedNumberOfVersions = 3 + + if err = c.ParseFlags([]string{"--merge", indexForMergeYAML}); err != nil { + t.Error(err) + } + if err := c.RunE(c, []string{dir}); err != nil { + t.Error(err) + } + + index, err = repo.LoadIndexFile(destIndex) + if err != nil { + t.Fatal(err) + } + + if len(index.Entries) != expectedNumberOfEntries { + t.Errorf("expected %d entries, got %d: %#v", + expectedNumberOfEntries, len(index.Entries), index.Entries) + } + + vs = index.Entries["compressedchart"] + if len(vs) != expectedNumberOfVersions { + t.Errorf("expected %d versions, got %d: %#v", + expectedNumberOfVersions, len(vs), vs) + } + + expectedVersion = "0.3.0" + if vs[0].Version != expectedVersion { + t.Errorf("expected %q, got %q", expectedVersion, vs[0].Version) + } + + jsonIndex, err = repo.LoadIndexJSONFile(destJSONIndex) + if err != nil { + t.Fatal(err) + } + + if len(jsonIndex.Entries) != expectedNumberOfEntries { + t.Errorf("expected %d entry in json index, got %d: %#v", + expectedNumberOfEntries, len(jsonIndex.Entries), jsonIndex.Entries) + } + + versionsInJSONIndex = jsonIndex.Entries["compressedchart"] + if len(versionsInJSONIndex) != expectedNumberOfVersions { + t.Errorf("expected %d versions in json index, got %d: %#v", + expectedNumberOfVersions, len(versionsInJSONIndex), versionsInJSONIndex) + } + + expectedVersionInJSONIndex = "0.3.0" + if versionsInJSONIndex[0].Version != expectedVersionInJSONIndex { + t.Errorf("expected %q in json index, got %q", + expectedVersionInJSONIndex, versionsInJSONIndex[0].Version) + } + + // Test with `--merge-json-index` + // cleanup index.yaml and index.json as it's not needed. it will be created + // by the command + if err := os.Remove(destIndex); err != nil { + t.Fatal(err) + } + if err := os.Remove(destJSONIndex); err != nil { + t.Fatal(err) + } + + buf = bytes.NewBuffer(nil) + c = newRepoIndexCmd(buf) + expectedNumberOfEntries = 2 + expectedNumberOfVersions = 3 + + if err = c.ParseFlags([]string{"--merge-json-index", indexForMergeJSON}); err != nil { + t.Error(err) + } + if err := c.RunE(c, []string{dir}); err != nil { + t.Error(err) + } + + index, err = repo.LoadIndexFile(destIndex) + if err != nil { + t.Fatal(err) + } + + if len(index.Entries) != expectedNumberOfEntries { + t.Errorf("expected %d entries, got %d: %#v", + expectedNumberOfEntries, len(index.Entries), index.Entries) + } + + vs = index.Entries["compressedchart"] + if len(vs) != expectedNumberOfVersions { + t.Errorf("expected %d versions, got %d: %#v", + expectedNumberOfVersions, len(vs), vs) + } + + expectedVersion = "0.3.0" + if vs[0].Version != expectedVersion { + t.Errorf("expected %q, got %q", expectedVersion, vs[0].Version) + } + + jsonIndex, err = repo.LoadIndexJSONFile(destJSONIndex) + if err != nil { + t.Fatal(err) + } + + if len(jsonIndex.Entries) != expectedNumberOfEntries { + t.Errorf("expected %d entry in json index, got %d: %#v", + expectedNumberOfEntries, len(jsonIndex.Entries), jsonIndex.Entries) + } + + versionsInJSONIndex = jsonIndex.Entries["compressedchart"] + if len(versionsInJSONIndex) != expectedNumberOfVersions { + t.Errorf("expected %d versions in json index, got %d: %#v", + expectedNumberOfVersions, len(versionsInJSONIndex), versionsInJSONIndex) + } + + expectedVersionInJSONIndex = "0.3.0" + if versionsInJSONIndex[0].Version != expectedVersionInJSONIndex { + t.Errorf("expected %q in json index, got %q", + expectedVersionInJSONIndex, versionsInJSONIndex[0].Version) + } + + // test that index.yaml and index.json gets generated on + // merge even when given index.yaml file doesn't doesn't exist + if err := os.Remove(destIndex); err != nil { + t.Fatal(err) + } + + if err := os.Remove(destJSONIndex); err != nil { + t.Fatal(err) + } + + buf = bytes.NewBuffer(nil) + c = newRepoIndexCmd(buf) + expectedNumberOfEntries = 2 + expectedNumberOfVersions = 1 + + if err = c.ParseFlags([]string{"--merge", destIndex}); err != nil { + t.Error(err) + } if err := c.RunE(c, []string{dir}); err != nil { t.Error(err) } @@ -95,13 +281,17 @@ func TestRepoIndexCmd(t *testing.T) { t.Fatal(err) } - if len(index.Entries) != 2 { - t.Errorf("expected 2 entries, got %d: %#v", len(index.Entries), index.Entries) + // verify it didn't create an empty index.yaml or empty index.json + // and the merged happened + if len(index.Entries) != expectedNumberOfEntries { + t.Errorf("expected %d entries, got %d: %#v", + expectedNumberOfEntries, len(index.Entries), index.Entries) } vs = index.Entries["compressedchart"] - if len(vs) != 3 { - t.Errorf("expected 3 versions, got %d: %#v", len(vs), vs) + if len(vs) != expectedNumberOfVersions { + t.Errorf("expected %d versions, got %d: %#v", + expectedNumberOfVersions, len(vs), vs) } expectedVersion = "0.3.0" @@ -109,12 +299,46 @@ func TestRepoIndexCmd(t *testing.T) { t.Errorf("expected %q, got %q", expectedVersion, vs[0].Version) } - // test that index.yaml gets generated on merge even when it doesn't exist + jsonIndex, err = repo.LoadIndexJSONFile(destJSONIndex) + if err != nil { + t.Fatal(err) + } + + if len(jsonIndex.Entries) != expectedNumberOfEntries { + t.Errorf("expected %d entry in json index, got %d: %#v", + expectedNumberOfEntries, len(jsonIndex.Entries), jsonIndex.Entries) + } + + versionsInJSONIndex = jsonIndex.Entries["compressedchart"] + if len(versionsInJSONIndex) != expectedNumberOfVersions { + t.Errorf("expected %d versions in json index, got %d: %#v", + expectedNumberOfVersions, len(versionsInJSONIndex), versionsInJSONIndex) + } + + expectedVersionInJSONIndex = "0.3.0" + if versionsInJSONIndex[0].Version != expectedVersionInJSONIndex { + t.Errorf("expected %q in json index, got %q", + expectedVersionInJSONIndex, versionsInJSONIndex[0].Version) + } + + // test that index.yaml and index.json gets generated on + // merge even when given index.json file doesn't doesn't exist if err := os.Remove(destIndex); err != nil { t.Fatal(err) } - c.ParseFlags([]string{"--merge", destIndex}) + if err := os.Remove(destJSONIndex); err != nil { + t.Fatal(err) + } + + buf = bytes.NewBuffer(nil) + c = newRepoIndexCmd(buf) + expectedNumberOfEntries = 2 + expectedNumberOfVersions = 1 + + if err = c.ParseFlags([]string{"--merge-json-index", destJSONIndex}); err != nil { + t.Error(err) + } if err := c.RunE(c, []string{dir}); err != nil { t.Error(err) } @@ -124,20 +348,72 @@ func TestRepoIndexCmd(t *testing.T) { t.Fatal(err) } - // verify it didn't create an empty index.yaml and the merged happened - if len(index.Entries) != 2 { - t.Errorf("expected 2 entries, got %d: %#v", len(index.Entries), index.Entries) + // verify it didn't create an empty index.yaml or empty index.json + // and the merged happened + if len(index.Entries) != expectedNumberOfEntries { + t.Errorf("expected %d entries, got %d: %#v", + expectedNumberOfEntries, len(index.Entries), index.Entries) } vs = index.Entries["compressedchart"] - if len(vs) != 1 { - t.Errorf("expected 1 versions, got %d: %#v", len(vs), vs) + if len(vs) != expectedNumberOfVersions { + t.Errorf("expected %d versions, got %d: %#v", + expectedNumberOfVersions, len(vs), vs) } expectedVersion = "0.3.0" if vs[0].Version != expectedVersion { t.Errorf("expected %q, got %q", expectedVersion, vs[0].Version) } + + jsonIndex, err = repo.LoadIndexJSONFile(destJSONIndex) + if err != nil { + t.Fatal(err) + } + + if len(jsonIndex.Entries) != expectedNumberOfEntries { + t.Errorf("expected %d entry in json index, got %d: %#v", + expectedNumberOfEntries, len(jsonIndex.Entries), jsonIndex.Entries) + } + + versionsInJSONIndex = jsonIndex.Entries["compressedchart"] + if len(versionsInJSONIndex) != expectedNumberOfVersions { + t.Errorf("expected %d versions in json index, got %d: %#v", + expectedNumberOfVersions, len(versionsInJSONIndex), versionsInJSONIndex) + } + + expectedVersionInJSONIndex = "0.3.0" + if versionsInJSONIndex[0].Version != expectedVersionInJSONIndex { + t.Errorf("expected %q in json index, got %q", + expectedVersionInJSONIndex, versionsInJSONIndex[0].Version) + } + // }) +} + +func TestRepoIndexCmd_Another(t *testing.T) { + t.Run("passing both --merge and --merge-json-index should fail", func(t *testing.T) { + dir := ensure.TempDir(t) + buf := bytes.NewBuffer(nil) + c := newRepoIndexCmd(buf) + + destIndex := filepath.Join(dir, "index.yaml") + destJSONIndex := filepath.Join(dir, "index.json") + + if err := c.ParseFlags([]string{"--merge", destIndex, + "--merge-json-index", destJSONIndex}); err != nil { + t.Error(err) + } + err := c.RunE(c, []string{dir}) + if err == nil { + t.Error("expected error but got nil") + return + } + + expectedError := "only one of --merge and --merge-json-index can be passed" + if !strings.Contains(err.Error(), expectedError) { + t.Errorf("expected error '%s' but got '%s'", expectedError, err.Error()) + } + }) } func linkOrCopy(old, new string) error { @@ -148,14 +424,14 @@ func linkOrCopy(old, new string) error { return nil } -func copyFile(dst, src string) error { - i, err := os.Open(dst) +func copyFile(src, dest string) error { + i, err := os.Open(src) if err != nil { return err } defer i.Close() - o, err := os.Create(src) + o, err := os.Create(dest) if err != nil { return err } diff --git a/pkg/repo/index.go b/pkg/repo/index.go index 36386665e..9f9f9ee6b 100644 --- a/pkg/repo/index.go +++ b/pkg/repo/index.go @@ -17,6 +17,7 @@ limitations under the License. package repo import ( + "encoding/json" "io/ioutil" "os" "path" @@ -100,6 +101,15 @@ func LoadIndexFile(path string) (*IndexFile, error) { return loadIndex(b) } +// LoadIndexJSONFile takes a JSON file at the given path and returns an IndexFile object +func LoadIndexJSONFile(path string) (*IndexFile, error) { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + return loadIndexJSON(b) +} + // Add adds a file to the index // This can leave the index in an unsorted state func (i IndexFile) Add(md *chart.Metadata, filename, baseURL, digest string) { @@ -189,7 +199,7 @@ func (i IndexFile) Get(name, version string) (*ChartVersion, error) { return nil, errors.Errorf("no chart version found for %s-%s", name, version) } -// WriteFile writes an index file to the given destination path. +// WriteFile writes an index.yaml file to the given destination path. // // The mode on the file is set to 'mode'. func (i IndexFile) WriteFile(dest string, mode os.FileMode) error { @@ -200,6 +210,17 @@ func (i IndexFile) WriteFile(dest string, mode os.FileMode) error { return ioutil.WriteFile(dest, b, mode) } +// WriteJSONFile writes an index.json file to the given destination path. +// +// The mode on the file is set to 'mode'. +func (i IndexFile) WriteJSONFile(dest string, mode os.FileMode) error { + b, err := json.Marshal(i) + if err != nil { + return err + } + return ioutil.WriteFile(dest, b, mode) +} + // Merge merges the given index file into this index. // // This merges by name and version. @@ -288,3 +309,18 @@ func loadIndex(data []byte) (*IndexFile, error) { } return i, nil } + +// loadIndexJSON loads an index.json file and does minimal validity checking. +// +// This will fail if API Version is not set (ErrNoAPIVersion) or if the unmarshal fails. +func loadIndexJSON(data []byte) (*IndexFile, error) { + i := &IndexFile{} + if err := json.Unmarshal(data, i); err != nil { + return i, err + } + i.SortEntries() + if i.APIVersion == "" { + return i, ErrNoAPIVersion + } + return i, nil +} diff --git a/pkg/repo/index_test.go b/pkg/repo/index_test.go index 5dbd5e551..6e03b8e60 100644 --- a/pkg/repo/index_test.go +++ b/pkg/repo/index_test.go @@ -36,6 +36,7 @@ import ( const ( testfile = "testdata/local-index.yaml" + testjsonfile = "testdata/local-index.json" unorderedTestfile = "testdata/local-index-unordered.yaml" testRepo = "test-repo" ) @@ -103,6 +104,26 @@ func TestLoadIndexFile(t *testing.T) { verifyLocalIndex(t, i) } +func TestLoadIndexJSON(t *testing.T) { + b, err := ioutil.ReadFile(testjsonfile) + if err != nil { + t.Fatal(err) + } + i, err := loadIndexJSON(b) + if err != nil { + t.Fatal(err) + } + verifyLocalIndex(t, i) +} + +func TestLoadIndexJSONFile(t *testing.T) { + i, err := LoadIndexJSONFile(testjsonfile) + if err != nil { + t.Fatal(err) + } + verifyLocalIndex(t, i) +} + func TestLoadUnorderedIndex(t *testing.T) { b, err := ioutil.ReadFile(unorderedTestfile) if err != nil { diff --git a/pkg/repo/testdata/local-index.json b/pkg/repo/testdata/local-index.json new file mode 100644 index 000000000..a35bf4c76 --- /dev/null +++ b/pkg/repo/testdata/local-index.json @@ -0,0 +1,69 @@ +{ + "apiVersion": "v1", + "entries": { + "nginx": [ + { + "urls": [ + "https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz" + ], + "name": "nginx", + "description": "string", + "version": "0.2.0", + "home": "https://github.com/something/else", + "digest": "sha256:1234567890abcdef", + "keywords": [ + "popular", + "web server", + "proxy" + ] + }, + { + "urls": [ + "https://kubernetes-charts.storage.googleapis.com/nginx-0.1.0.tgz" + ], + "name": "nginx", + "description": "string", + "version": "0.1.0", + "home": "https://github.com/something", + "digest": "sha256:1234567890abcdef", + "keywords": [ + "popular", + "web server", + "proxy" + ] + } + ], + "alpine": [ + { + "urls": [ + "https://kubernetes-charts.storage.googleapis.com/alpine-1.0.0.tgz", + "http://storage2.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz" + ], + "name": "alpine", + "description": "string", + "version": "1.0.0", + "home": "https://github.com/something", + "keywords": [ + "linux", + "alpine", + "small", + "sumtin" + ], + "digest": "sha256:1234567890abcdef" + } + ], + "chartWithNoURL": [ + { + "name": "chartWithNoURL", + "description": "string", + "version": "1.0.0", + "home": "https://github.com/something", + "keywords": [ + "small", + "sumtin" + ], + "digest": "sha256:1234567890abcdef" + } + ] + } +}