msb-public
/
helm
mirror of https://github.com/helm/helm
1
0
Code Issues Projects Releases Wiki Activity

feat(repo_index): generate index.json too for repo index command

Browse Source 
Signed-off-by: Karuppiah Natarajan <karuppiah7890@gmail.com>
pull/7510/head
Karuppiah Natarajan 6 years ago
parent c12a9aee02
commit aca4fb06ce
No known key found for this signature in database
GPG Key ID: C674A28337662A96
5 changed files with 496 additions and 44 deletions
Show Stats Download Patch File Download Diff File

72
cmd/helm/repo_index.go
Unescape View File

@ -31,11 +31,12 @@ import (
const repoIndexDesc = `
Read the current directory and generate an index file based on the charts found.
This tool is used for creating an 'index.yaml' file for a chart repository. To
This tool is used for creating an 'index.yaml' and 'index.json' files for a chart repository. To
set an absolute URL to the charts, use '--url' flag.
To merge the generated index with an existing index file, use the '--merge'
flag. In this case, the charts found in the current directory will be merged
flag for 'index.yaml' file and '--merge-json-index' for 'index.json' file. Only one of the
flags can be passed at once. In this case, the charts found in the current directory will be merged
into the existing index, with local charts taking priority over existing charts.
`
@ -43,6 +44,7 @@ type repoIndexOptions struct {
dir string
url string
merge string
mergeJSONIndex string
}
func newRepoIndexCmd(out io.Writer) *cobra.Command {
@ -61,29 +63,66 @@ func newRepoIndexCmd(out io.Writer) *cobra.Command {
f := cmd.Flags()
f.StringVar(&o.url, "url", "", "url of chart repository")
f.StringVar(&o.merge, "merge", "", "merge the generated index into the given index")
f.StringVar(&o.merge, "merge", "", "merge the generated index into the given index.yaml")
f.StringVar(&o.mergeJSONIndex, "merge-json-index", "", "merge the generated index into the given index.json")
return cmd
}
func (i *repoIndexOptions) run(out io.Writer) error {
if i.merge != "" && i.mergeJSONIndex != "" {
return errors.New("only one of --merge and --merge-json-index can be passed")
}
path, err := filepath.Abs(i.dir)
if err != nil {
return err
}
return index(path, i.url, i.merge)
return index(path, i.url, i.merge, i.mergeJSONIndex)
}
func index(dir, url, mergeTo string) error {
out := filepath.Join(dir, "index.yaml")
func index(dir, url, mergeTo, mergeToJSON string) error {
yamlIndex := filepath.Join(dir, "index.yaml")
jsonIndex := filepath.Join(dir, "index.json")
i, err := repo.IndexDirectory(dir, url)
if err != nil {
return err
}
if mergeTo != "" {
if mergeTo != "" || mergeToJSON != "" {
if err = mergeToIndex(i, mergeTo, mergeToJSON); err != nil {
return err
}
}
i.SortEntries()
err = i.WriteFile(yamlIndex, 0644)
if err != nil {
return err
}
return i.WriteJSONFile(jsonIndex, 0644)
}
func mergeToIndex(i *repo.IndexFile, mergeTo, mergeToJSON string) error {
// if index.yaml is missing then create an empty one to merge into
var i2 *repo.IndexFile
var err error
if mergeTo != "" {
i2, err = loadIndexForMerge(mergeTo)
if err != nil {
return err
}
} else if mergeToJSON != "" {
i2, err = loadIndexJSONForMerge(mergeToJSON)
if err != nil {
return err
}
}
i.Merge(i2)
return nil
}
func loadIndexForMerge(mergeTo string) (*repo.IndexFile, error) {
var i2 *repo.IndexFile
if _, err := os.Stat(mergeTo); os.IsNotExist(err) {
i2 = repo.NewIndexFile()
@ -91,11 +130,22 @@ func index(dir, url, mergeTo string) error {
} else {
i2, err = repo.LoadIndexFile(mergeTo)
if err != nil {
return errors.Wrap(err, "merge failed")
return nil, errors.Wrap(err, "merge failed")
}
}
i.Merge(i2)
return i2, nil
}
i.SortEntries()
return i.WriteFile(out, 0644)
func loadIndexJSONForMerge(mergeToJSON string) (*repo.IndexFile, error) {
var i2 *repo.IndexFile
if _, err := os.Stat(mergeToJSON); os.IsNotExist(err) {
i2 = repo.NewIndexFile()
i2.WriteJSONFile(mergeToJSON, 0644)
} else {
i2, err = repo.LoadIndexJSONFile(mergeToJSON)
if err != nil {
return nil, errors.Wrap(err, "merge failed")
}
}
return i2, nil
}

318
cmd/helm/repo_index_test.go
Unescape View File

@ -21,6 +21,7 @@ import (
"io"
"os"
"path/filepath"
"strings"
"testing"
"helm.sh/helm/v3/internal/test/ensure"
@ -28,7 +29,7 @@ import (
)
func TestRepoIndexCmd(t *testing.T) {
// t.Run("helm repo index command", func(t *testing.T) {
dir := ensure.TempDir(t)
comp := filepath.Join(dir, "compressedchart-0.1.0.tgz")
@ -42,6 +43,8 @@ func TestRepoIndexCmd(t *testing.T) {
buf := bytes.NewBuffer(nil)
c := newRepoIndexCmd(buf)
expectedNumberOfEntries := 1
expectedNumberOfVersions := 2
if err := c.RunE(c, []string{dir}); err != nil {
t.Error(err)
@ -54,13 +57,15 @@ func TestRepoIndexCmd(t *testing.T) {
t.Fatal(err)
}
if len(index.Entries) != 1 {
t.Errorf("expected 1 entry, got %d: %#v", len(index.Entries), index.Entries)
if len(index.Entries) != expectedNumberOfEntries {
t.Errorf("expected %d entry, got %d: %#v",
expectedNumberOfEntries, len(index.Entries), index.Entries)
}
vs := index.Entries["compressedchart"]
if len(vs) != 2 {
t.Errorf("expected 2 versions, got %d: %#v", len(vs), vs)
if len(vs) != expectedNumberOfVersions {
t.Errorf("expected %d versions, got %d: %#v",
expectedNumberOfVersions, len(vs), vs)
}
expectedVersion := "0.2.0"
@ -68,8 +73,52 @@ func TestRepoIndexCmd(t *testing.T) {
t.Errorf("expected %q, got %q", expectedVersion, vs[0].Version)
}
// Test with `--merge`
// Test creation of index.json
destJSONIndex := filepath.Join(dir, "index.json")
jsonIndex, err := repo.LoadIndexJSONFile(destJSONIndex)
if err != nil {
t.Fatal(err)
}
if len(jsonIndex.Entries) != expectedNumberOfEntries {
t.Errorf("expected %d entry in json index, got %d: %#v",
expectedNumberOfEntries, len(jsonIndex.Entries), jsonIndex.Entries)
}
versionsInJSONIndex := jsonIndex.Entries["compressedchart"]
if len(versionsInJSONIndex) != expectedNumberOfVersions {
t.Errorf("expected %d versions in json index, got %d: %#v",
expectedNumberOfVersions, len(versionsInJSONIndex), versionsInJSONIndex)
}
expectedVersionInJSONIndex := "0.2.0"
if versionsInJSONIndex[0].Version != expectedVersionInJSONIndex {
t.Errorf("expected %q in json index, got %q",
expectedVersionInJSONIndex, versionsInJSONIndex[0].Version)
}
// save a copy of index.json to test --merge-json-index later
indexForMergeJSON := filepath.Join(dir, "indexForMerge.json")
if err = copyFile(destJSONIndex, indexForMergeJSON); err != nil {
t.Fatal(err)
}
// save a copy of index.yaml to test --merge now
indexForMergeYAML := filepath.Join(dir, "indexForMerge.yaml")
if err = copyFile(destIndex, indexForMergeYAML); err != nil {
t.Fatal(err)
}
// Test with `--merge`
// cleanup index.json and index.yaml as it's not needed. it will be created
// by the command
if err := os.Remove(destJSONIndex); err != nil {
t.Fatal(err)
}
if err := os.Remove(destIndex); err != nil {
t.Fatal(err)
}
// Remove first two charts.
if err := os.Remove(comp); err != nil {
t.Fatal(err)
@ -85,7 +134,14 @@ func TestRepoIndexCmd(t *testing.T) {
t.Fatal(err)
}
c.ParseFlags([]string{"--merge", destIndex})
buf = bytes.NewBuffer(nil)
c = newRepoIndexCmd(buf)
expectedNumberOfEntries = 2
expectedNumberOfVersions = 3
if err = c.ParseFlags([]string{"--merge", indexForMergeYAML}); err != nil {
t.Error(err)
}
if err := c.RunE(c, []string{dir}); err != nil {
t.Error(err)
}
@ -95,13 +151,15 @@ func TestRepoIndexCmd(t *testing.T) {
t.Fatal(err)
}
if len(index.Entries) != 2 {
t.Errorf("expected 2 entries, got %d: %#v", len(index.Entries), index.Entries)
if len(index.Entries) != expectedNumberOfEntries {
t.Errorf("expected %d entries, got %d: %#v",
expectedNumberOfEntries, len(index.Entries), index.Entries)
}
vs = index.Entries["compressedchart"]
if len(vs) != 3 {
t.Errorf("expected 3 versions, got %d: %#v", len(vs), vs)
if len(vs) != expectedNumberOfVersions {
t.Errorf("expected %d versions, got %d: %#v",
expectedNumberOfVersions, len(vs), vs)
}
expectedVersion = "0.3.0"
@ -109,12 +167,46 @@ func TestRepoIndexCmd(t *testing.T) {
t.Errorf("expected %q, got %q", expectedVersion, vs[0].Version)
}
// test that index.yaml gets generated on merge even when it doesn't exist
jsonIndex, err = repo.LoadIndexJSONFile(destJSONIndex)
if err != nil {
t.Fatal(err)
}
if len(jsonIndex.Entries) != expectedNumberOfEntries {
t.Errorf("expected %d entry in json index, got %d: %#v",
expectedNumberOfEntries, len(jsonIndex.Entries), jsonIndex.Entries)
}
versionsInJSONIndex = jsonIndex.Entries["compressedchart"]
if len(versionsInJSONIndex) != expectedNumberOfVersions {
t.Errorf("expected %d versions in json index, got %d: %#v",
expectedNumberOfVersions, len(versionsInJSONIndex), versionsInJSONIndex)
}
expectedVersionInJSONIndex = "0.3.0"
if versionsInJSONIndex[0].Version != expectedVersionInJSONIndex {
t.Errorf("expected %q in json index, got %q",
expectedVersionInJSONIndex, versionsInJSONIndex[0].Version)
}
// Test with `--merge-json-index`
// cleanup index.yaml and index.json as it's not needed. it will be created
// by the command
if err := os.Remove(destIndex); err != nil {
t.Fatal(err)
}
if err := os.Remove(destJSONIndex); err != nil {
t.Fatal(err)
}
c.ParseFlags([]string{"--merge", destIndex})
buf = bytes.NewBuffer(nil)
c = newRepoIndexCmd(buf)
expectedNumberOfEntries = 2
expectedNumberOfVersions = 3
if err = c.ParseFlags([]string{"--merge-json-index", indexForMergeJSON}); err != nil {
t.Error(err)
}
if err := c.RunE(c, []string{dir}); err != nil {
t.Error(err)
}
@ -124,20 +216,204 @@ func TestRepoIndexCmd(t *testing.T) {
t.Fatal(err)
}
// verify it didn't create an empty index.yaml and the merged happened
if len(index.Entries) != 2 {
t.Errorf("expected 2 entries, got %d: %#v", len(index.Entries), index.Entries)
if len(index.Entries) != expectedNumberOfEntries {
t.Errorf("expected %d entries, got %d: %#v",
expectedNumberOfEntries, len(index.Entries), index.Entries)
}
vs = index.Entries["compressedchart"]
if len(vs) != 1 {
t.Errorf("expected 1 versions, got %d: %#v", len(vs), vs)
if len(vs) != expectedNumberOfVersions {
t.Errorf("expected %d versions, got %d: %#v",
expectedNumberOfVersions, len(vs), vs)
}
expectedVersion = "0.3.0"
if vs[0].Version != expectedVersion {
t.Errorf("expected %q, got %q", expectedVersion, vs[0].Version)
}
jsonIndex, err = repo.LoadIndexJSONFile(destJSONIndex)
if err != nil {
t.Fatal(err)
}
if len(jsonIndex.Entries) != expectedNumberOfEntries {
t.Errorf("expected %d entry in json index, got %d: %#v",
expectedNumberOfEntries, len(jsonIndex.Entries), jsonIndex.Entries)
}
versionsInJSONIndex = jsonIndex.Entries["compressedchart"]
if len(versionsInJSONIndex) != expectedNumberOfVersions {
t.Errorf("expected %d versions in json index, got %d: %#v",
expectedNumberOfVersions, len(versionsInJSONIndex), versionsInJSONIndex)
}
expectedVersionInJSONIndex = "0.3.0"
if versionsInJSONIndex[0].Version != expectedVersionInJSONIndex {
t.Errorf("expected %q in json index, got %q",
expectedVersionInJSONIndex, versionsInJSONIndex[0].Version)
}
// test that index.yaml and index.json gets generated on
// merge even when given index.yaml file doesn't doesn't exist
if err := os.Remove(destIndex); err != nil {
t.Fatal(err)
}
if err := os.Remove(destJSONIndex); err != nil {
t.Fatal(err)
}
buf = bytes.NewBuffer(nil)
c = newRepoIndexCmd(buf)
expectedNumberOfEntries = 2
expectedNumberOfVersions = 1
if err = c.ParseFlags([]string{"--merge", destIndex}); err != nil {
t.Error(err)
}
if err := c.RunE(c, []string{dir}); err != nil {
t.Error(err)
}
index, err = repo.LoadIndexFile(destIndex)
if err != nil {
t.Fatal(err)
}
// verify it didn't create an empty index.yaml or empty index.json
// and the merged happened
if len(index.Entries) != expectedNumberOfEntries {
t.Errorf("expected %d entries, got %d: %#v",
expectedNumberOfEntries, len(index.Entries), index.Entries)
}
vs = index.Entries["compressedchart"]
if len(vs) != expectedNumberOfVersions {
t.Errorf("expected %d versions, got %d: %#v",
expectedNumberOfVersions, len(vs), vs)
}
expectedVersion = "0.3.0"
if vs[0].Version != expectedVersion {
t.Errorf("expected %q, got %q", expectedVersion, vs[0].Version)
}
jsonIndex, err = repo.LoadIndexJSONFile(destJSONIndex)
if err != nil {
t.Fatal(err)
}
if len(jsonIndex.Entries) != expectedNumberOfEntries {
t.Errorf("expected %d entry in json index, got %d: %#v",
expectedNumberOfEntries, len(jsonIndex.Entries), jsonIndex.Entries)
}
versionsInJSONIndex = jsonIndex.Entries["compressedchart"]
if len(versionsInJSONIndex) != expectedNumberOfVersions {
t.Errorf("expected %d versions in json index, got %d: %#v",
expectedNumberOfVersions, len(versionsInJSONIndex), versionsInJSONIndex)
}
expectedVersionInJSONIndex = "0.3.0"
if versionsInJSONIndex[0].Version != expectedVersionInJSONIndex {
t.Errorf("expected %q in json index, got %q",
expectedVersionInJSONIndex, versionsInJSONIndex[0].Version)
}
// test that index.yaml and index.json gets generated on
// merge even when given index.json file doesn't doesn't exist
if err := os.Remove(destIndex); err != nil {
t.Fatal(err)
}
if err := os.Remove(destJSONIndex); err != nil {
t.Fatal(err)
}
buf = bytes.NewBuffer(nil)
c = newRepoIndexCmd(buf)
expectedNumberOfEntries = 2
expectedNumberOfVersions = 1
if err = c.ParseFlags([]string{"--merge-json-index", destJSONIndex}); err != nil {
t.Error(err)
}
if err := c.RunE(c, []string{dir}); err != nil {
t.Error(err)
}
index, err = repo.LoadIndexFile(destIndex)
if err != nil {
t.Fatal(err)
}
// verify it didn't create an empty index.yaml or empty index.json
// and the merged happened
if len(index.Entries) != expectedNumberOfEntries {
t.Errorf("expected %d entries, got %d: %#v",
expectedNumberOfEntries, len(index.Entries), index.Entries)
}
vs = index.Entries["compressedchart"]
if len(vs) != expectedNumberOfVersions {
t.Errorf("expected %d versions, got %d: %#v",
expectedNumberOfVersions, len(vs), vs)
}
expectedVersion = "0.3.0"
if vs[0].Version != expectedVersion {
t.Errorf("expected %q, got %q", expectedVersion, vs[0].Version)
}
jsonIndex, err = repo.LoadIndexJSONFile(destJSONIndex)
if err != nil {
t.Fatal(err)
}
if len(jsonIndex.Entries) != expectedNumberOfEntries {
t.Errorf("expected %d entry in json index, got %d: %#v",
expectedNumberOfEntries, len(jsonIndex.Entries), jsonIndex.Entries)
}
versionsInJSONIndex = jsonIndex.Entries["compressedchart"]
if len(versionsInJSONIndex) != expectedNumberOfVersions {
t.Errorf("expected %d versions in json index, got %d: %#v",
expectedNumberOfVersions, len(versionsInJSONIndex), versionsInJSONIndex)
}
expectedVersionInJSONIndex = "0.3.0"
if versionsInJSONIndex[0].Version != expectedVersionInJSONIndex {
t.Errorf("expected %q in json index, got %q",
expectedVersionInJSONIndex, versionsInJSONIndex[0].Version)
}
// })
}
func TestRepoIndexCmd_Another(t *testing.T) {
t.Run("passing both --merge and --merge-json-index should fail", func(t *testing.T) {
dir := ensure.TempDir(t)
buf := bytes.NewBuffer(nil)
c := newRepoIndexCmd(buf)
destIndex := filepath.Join(dir, "index.yaml")
destJSONIndex := filepath.Join(dir, "index.json")
if err := c.ParseFlags([]string{"--merge", destIndex,
"--merge-json-index", destJSONIndex}); err != nil {
t.Error(err)
}
err := c.RunE(c, []string{dir})
if err == nil {
t.Error("expected error but got nil")
return
}
expectedError := "only one of --merge and --merge-json-index can be passed"
if !strings.Contains(err.Error(), expectedError) {
t.Errorf("expected error '%s' but got '%s'", expectedError, err.Error())
}
})
}
func linkOrCopy(old, new string) error {
@ -148,14 +424,14 @@ func linkOrCopy(old, new string) error {
return nil
}
func copyFile(dst, src string) error {
i, err := os.Open(dst)
func copyFile(src, dest string) error {
i, err := os.Open(src)
if err != nil {
return err
}
defer i.Close()
o, err := os.Create(src)
o, err := os.Create(dest)
if err != nil {
return err
}

38
pkg/repo/index.go
Unescape View File

@ -17,6 +17,7 @@ limitations under the License.
package repo
import (
"encoding/json"
"io/ioutil"
"os"
"path"
@ -100,6 +101,15 @@ func LoadIndexFile(path string) (*IndexFile, error) {
return loadIndex(b)
}
// LoadIndexJSONFile takes a JSON file at the given path and returns an IndexFile object
func LoadIndexJSONFile(path string) (*IndexFile, error) {
b, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
return loadIndexJSON(b)
}
// Add adds a file to the index
// This can leave the index in an unsorted state
func (i IndexFile) Add(md *chart.Metadata, filename, baseURL, digest string) {
@ -189,7 +199,7 @@ func (i IndexFile) Get(name, version string) (*ChartVersion, error) {
return nil, errors.Errorf("no chart version found for %s-%s", name, version)
}
// WriteFile writes an index file to the given destination path.
// WriteFile writes an index.yaml file to the given destination path.
//
// The mode on the file is set to 'mode'.
func (i IndexFile) WriteFile(dest string, mode os.FileMode) error {
@ -200,6 +210,17 @@ func (i IndexFile) WriteFile(dest string, mode os.FileMode) error {
return ioutil.WriteFile(dest, b, mode)
}
// WriteJSONFile writes an index.json file to the given destination path.
//
// The mode on the file is set to 'mode'.
func (i IndexFile) WriteJSONFile(dest string, mode os.FileMode) error {
b, err := json.Marshal(i)
if err != nil {
return err
}
return ioutil.WriteFile(dest, b, mode)
}
// Merge merges the given index file into this index.
//
// This merges by name and version.
@ -288,3 +309,18 @@ func loadIndex(data []byte) (*IndexFile, error) {
}
return i, nil
}
// loadIndexJSON loads an index.json file and does minimal validity checking.
//
// This will fail if API Version is not set (ErrNoAPIVersion) or if the unmarshal fails.
func loadIndexJSON(data []byte) (*IndexFile, error) {
i := &IndexFile{}
if err := json.Unmarshal(data, i); err != nil {
return i, err
}
i.SortEntries()
if i.APIVersion == "" {
return i, ErrNoAPIVersion
}
return i, nil
}

21
pkg/repo/index_test.go
Unescape View File

@ -36,6 +36,7 @@ import (
const (
testfile = "testdata/local-index.yaml"
testjsonfile = "testdata/local-index.json"
unorderedTestfile = "testdata/local-index-unordered.yaml"
testRepo = "test-repo"
)
@ -103,6 +104,26 @@ func TestLoadIndexFile(t *testing.T) {
verifyLocalIndex(t, i)
}
func TestLoadIndexJSON(t *testing.T) {
b, err := ioutil.ReadFile(testjsonfile)
if err != nil {
t.Fatal(err)
}
i, err := loadIndexJSON(b)
if err != nil {
t.Fatal(err)
}
verifyLocalIndex(t, i)
}
func TestLoadIndexJSONFile(t *testing.T) {
i, err := LoadIndexJSONFile(testjsonfile)
if err != nil {
t.Fatal(err)
}
verifyLocalIndex(t, i)
}
func TestLoadUnorderedIndex(t *testing.T) {
b, err := ioutil.ReadFile(unorderedTestfile)
if err != nil {

69
pkg/repo/testdata/local-index.json vendored
Unescape View File

@ -0,0 +1,69 @@
{
"apiVersion": "v1",
"entries": {
"nginx": [
{
"urls": [
"https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz"
],
"name": "nginx",
"description": "string",
"version": "0.2.0",
"home": "https://github.com/something/else",
"digest": "sha256:1234567890abcdef",
"keywords": [
"popular",
"web server",
"proxy"
]
},
{
"urls": [
"https://kubernetes-charts.storage.googleapis.com/nginx-0.1.0.tgz"
],
"name": "nginx",
"description": "string",
"version": "0.1.0",
"home": "https://github.com/something",
"digest": "sha256:1234567890abcdef",
"keywords": [
"popular",
"web server",
"proxy"
]
}
],
"alpine": [
{
"urls": [
"https://kubernetes-charts.storage.googleapis.com/alpine-1.0.0.tgz",
"http://storage2.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz"
],
"name": "alpine",
"description": "string",
"version": "1.0.0",
"home": "https://github.com/something",
"keywords": [
"linux",
"alpine",
"small",
"sumtin"
],
"digest": "sha256:1234567890abcdef"
}
],
"chartWithNoURL": [
{
"name": "chartWithNoURL",
"description": "string",
"version": "1.0.0",
"home": "https://github.com/something",
"keywords": [
"small",
"sumtin"
],
"digest": "sha256:1234567890abcdef"
}
]
}
}
Write Preview
Loading…
Cancel
Save