msb-public
/
helm
mirror of https://github.com/helm/helm
1
0
Code Issues Projects Releases Wiki Activity

add support for index.json in repo add command

Browse Source
pull/7510/head
Karuppiah Natarajan 6 years ago
parent 670ad24cb2
commit 452dc71fd8
No known key found for this signature in database
GPG Key ID: C674A28337662A96
6 changed files with 340 additions and 57 deletions
Show Stats Download Patch File Download Diff File

61
cmd/helm/repo_add_test.go
Unescape View File

@ -104,6 +104,67 @@ func TestRepoAdd(t *testing.T) {
}
}
func TestRepoAdd_IndexJSON(t *testing.T) {
ts, err := repotest.NewTempServer("testdata/testserver/*.*")
if err != nil {
t.Fatal(err)
}
err = ts.ServeJSONIndex(true)
if err != nil {
t.Fatal(err)
}
err = ts.ServeYamlIndex(false)
if err != nil {
t.Fatal(err)
}
defer ts.Stop()
rootDir := ensure.TempDir(t)
repoFile := filepath.Join(rootDir, "repositories.yaml")
const testRepoName = "test-name"
o := &repoAddOptions{
name: testRepoName,
url: ts.URL(),
noUpdate: true,
repoFile: repoFile,
}
os.Setenv(xdg.CacheHomeEnvVar, rootDir)
if err := o.run(ioutil.Discard); err != nil {
t.Error(err)
}
f, err := repo.LoadFile(repoFile)
if err != nil {
t.Fatal(err)
}
if !f.Has(testRepoName) {
t.Errorf("%s was not successfully inserted into %s", testRepoName, repoFile)
}
idx := filepath.Join(helmpath.CachePath("repository"), helmpath.CacheIndexJSONFile(testRepoName))
if _, err := os.Stat(idx); os.IsNotExist(err) {
t.Errorf("Error cache json index file was not created for repository %s", testRepoName)
}
idx = filepath.Join(helmpath.CachePath("repository"), helmpath.CacheChartsFile(testRepoName))
if _, err := os.Stat(idx); os.IsNotExist(err) {
t.Errorf("Error cache charts file was not created for repository %s", testRepoName)
}
o.noUpdate = false
if err := o.run(ioutil.Discard); err != nil {
t.Errorf("Repository was not updated: %s", err)
}
if err := o.run(ioutil.Discard); err != nil {
t.Errorf("Duplicate repository name was added")
}
}
func TestRepoAddConcurrentGoRoutines(t *testing.T) {
const testName = "test-name"
repoFile := filepath.Join(ensure.TempDir(t), "repositories.yaml")

8
pkg/helmpath/home.go
Unescape View File

@ -34,6 +34,14 @@ func CacheIndexFile(name string) string {
return name + "index.yaml"
}
// CacheIndexJSONFile returns the path to an json index for the given named repository.
func CacheIndexJSONFile(name string) string {
if name != "" {
name += "-"
}
return name + "index.json"
}
// CacheChartsFile returns the path to a text file listing all the charts
// within the given named repository.
func CacheChartsFile(name string) string {

86
pkg/repo/chartrepo.go
Unescape View File

@ -113,12 +113,66 @@ func (r *ChartRepository) Load() error {
// DownloadIndexFile fetches the index from a repository.
func (r *ChartRepository) DownloadIndexFile() (string, error) {
parsedURL, err := url.Parse(r.Config.URL)
indexFile, err := r.downloadJSONOrYamlIndex()
if err != nil {
return "", err
}
parsedURL.RawPath = path.Join(parsedURL.RawPath, "index.yaml")
parsedURL.Path = path.Join(parsedURL.Path, "index.yaml")
// Create the chart list file in the cache directory
var charts strings.Builder
for name := range indexFile.Entries {
fmt.Fprintln(&charts, name)
}
chartsFile := filepath.Join(r.CachePath, helmpath.CacheChartsFile(r.Config.Name))
os.MkdirAll(filepath.Dir(chartsFile), 0755)
ioutil.WriteFile(chartsFile, []byte(charts.String()), 0644)
// Create the yaml index file in the cache directory
fname := filepath.Join(r.CachePath, helmpath.CacheIndexFile(r.Config.Name))
os.MkdirAll(filepath.Dir(fname), 0755)
// Create the json index file in the cache directory
jsonIndexFilePapth := filepath.Join(r.CachePath, helmpath.CacheIndexJSONFile(r.Config.Name))
os.MkdirAll(filepath.Dir(jsonIndexFilePapth), 0755)
_ = indexFile.WriteJSONFile(jsonIndexFilePapth, 0644)
return fname, indexFile.WriteFile(fname, 0644)
}
func (r *ChartRepository) downloadJSONOrYamlIndex() (*IndexFile, error) {
jsonIndexBytes, err := r.downloadIndexFile("index.json")
if err == nil {
// if json index has been downloaded without errors,
// all good, use it!
indexFile, err := loadIndexJSON(jsonIndexBytes)
if err == nil {
return indexFile, nil
}
}
// if json index download or parse has issues,
// swallow errors and fallback to using yaml index
yamlIndexBytes, err := r.downloadIndexFile("index.yaml")
if err != nil {
return nil, err
}
indexFile, err := loadIndex(yamlIndexBytes)
if err != nil {
return nil, err
}
return indexFile, nil
}
func (r *ChartRepository) downloadIndexFile(indexFileName string) ([]byte, error) {
parsedURL, err := url.Parse(r.Config.URL)
if err != nil {
return nil, err
}
parsedURL.RawPath = path.Join(parsedURL.RawPath, indexFileName)
parsedURL.Path = path.Join(parsedURL.Path, indexFileName)
indexURL := parsedURL.String()
// TODO add user-agent
@ -129,32 +183,14 @@ func (r *ChartRepository) DownloadIndexFile() (string, error) {
getter.WithBasicAuth(r.Config.Username, r.Config.Password),
)
if err != nil {
return "", err
}
index, err := ioutil.ReadAll(resp)
if err != nil {
return "", err
return nil, err
}
indexFile, err := loadIndex(index)
indexBytes, err := ioutil.ReadAll(resp)
if err != nil {
return "", err
return nil, err
}
// Create the chart list file in the cache directory
var charts strings.Builder
for name := range indexFile.Entries {
fmt.Fprintln(&charts, name)
}
chartsFile := filepath.Join(r.CachePath, helmpath.CacheChartsFile(r.Config.Name))
os.MkdirAll(filepath.Dir(chartsFile), 0755)
ioutil.WriteFile(chartsFile, []byte(charts.String()), 0644)
// Create the index file in the cache directory
fname := filepath.Join(r.CachePath, helmpath.CacheIndexFile(r.Config.Name))
os.MkdirAll(filepath.Dir(fname), 0755)
return fname, ioutil.WriteFile(fname, index, 0644)
return indexBytes, nil
}
// Index generates an index for the chart repository and writes an index.yaml file.

126
pkg/repo/index_test.go
Unescape View File

@ -170,8 +170,17 @@ func TestMerge(t *testing.T) {
}
func TestDownloadIndexFile(t *testing.T) {
t.Run("should download index file", func(t *testing.T) {
srv, err := startLocalServerForTests(nil)
t.Run("should download index.yaml file and create index.yaml and index.json", func(t *testing.T) {
fileBytes, err := ioutil.ReadFile("testdata/local-index.yaml")
if err != nil {
t.Fatal(err)
}
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/index.yaml" {
w.Write(fileBytes)
}
})
srv, err := startLocalServerForTests(handler)
if err != nil {
t.Fatal(err)
}
@ -185,37 +194,126 @@ func TestDownloadIndexFile(t *testing.T) {
t.Errorf("Problem creating chart repository from %s: %v", testRepo, err)
}
idx, err := r.DownloadIndexFile()
yamlIndexFilePath, err := r.DownloadIndexFile()
if err != nil {
t.Fatalf("Failed to download index file to %s: %#v", idx, err)
t.Fatalf("Failed to download index file: %#v", err)
}
if _, err := os.Stat(idx); err != nil {
if _, err := os.Stat(yamlIndexFilePath); err != nil {
t.Fatalf("error finding created index file: %#v", err)
}
b, err := ioutil.ReadFile(idx)
yamlIndexBytes, err := ioutil.ReadFile(yamlIndexFilePath)
if err != nil {
t.Fatalf("error reading index file: %#v", err)
t.Fatalf("error reading yaml index file: %#v", err)
}
i, err := loadIndex(b)
yamlIndex, err := loadIndex(yamlIndexBytes)
if err != nil {
t.Fatalf("Index %q failed to parse: %s", testfile, err)
t.Fatalf("Yaml index %q failed to parse: %s", testfile, err)
}
verifyLocalIndex(t, i)
verifyLocalIndex(t, yamlIndex)
jsonIndexFilePath := filepath.Join(helmpath.CachePath("repository"), helmpath.CacheIndexJSONFile(testRepo))
if _, err := os.Stat(jsonIndexFilePath); err != nil {
t.Fatalf("error finding created json index file: %#v", err)
}
jsonBytes, err := ioutil.ReadFile(jsonIndexFilePath)
if err != nil {
t.Fatalf("error reading json index file: %#v", err)
}
jsonIndex, err := loadIndexJSON(jsonBytes)
if err != nil {
t.Fatalf("Json index %q failed to parse: %s", testjsonfile, err)
}
verifyLocalIndex(t, jsonIndex)
// Check that charts file is also created
idx = filepath.Join(r.CachePath, helmpath.CacheChartsFile(r.Config.Name))
if _, err := os.Stat(idx); err != nil {
chartsFile := filepath.Join(r.CachePath, helmpath.CacheChartsFile(r.Config.Name))
if _, err := os.Stat(chartsFile); err != nil {
t.Fatalf("error finding created charts file: %#v", err)
}
b, err = ioutil.ReadFile(idx)
chartsFileBytes, err := ioutil.ReadFile(chartsFile)
if err != nil {
t.Fatalf("error reading charts file: %#v", err)
}
verifyLocalChartsFile(t, b, i)
verifyLocalChartsFile(t, chartsFileBytes, yamlIndex)
})
t.Run("should download index.json file and create index.json and index.yaml", func(t *testing.T) {
fileBytes, err := ioutil.ReadFile("testdata/local-index.json")
if err != nil {
t.Fatal(err)
}
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/index.json" {
w.Write(fileBytes)
}
})
srv, err := startLocalServerForTests(handler)
if err != nil {
t.Fatal(err)
}
defer srv.Close()
r, err := NewChartRepository(&Entry{
Name: testRepo,
URL: srv.URL,
}, getter.All(&cli.EnvSettings{}))
if err != nil {
t.Errorf("Problem creating chart repository from %s: %v", testRepo, err)
}
yamlIndexFilePath, err := r.DownloadIndexFile()
if err != nil {
t.Fatalf("Failed to download index file: %#v", err)
}
jsonIndexFilePath := filepath.Join(helmpath.CachePath("repository"), helmpath.CacheIndexJSONFile(testRepo))
if _, err := os.Stat(jsonIndexFilePath); err != nil {
t.Fatalf("error finding created json index file: %#v", err)
}
jsonBytes, err := ioutil.ReadFile(jsonIndexFilePath)
if err != nil {
t.Fatalf("error reading json index file: %#v", err)
}
jsonIndex, err := loadIndexJSON(jsonBytes)
if err != nil {
t.Fatalf("Json index %q failed to parse: %s", testjsonfile, err)
}
verifyLocalIndex(t, jsonIndex)
if _, err := os.Stat(yamlIndexFilePath); err != nil {
t.Fatalf("error finding created yaml index file: %#v", err)
}
yamlIndexBytes, err := ioutil.ReadFile(yamlIndexFilePath)
if err != nil {
t.Fatalf("error reading yaml index file: %#v", err)
}
yamlIndex, err := loadIndex(yamlIndexBytes)
if err != nil {
t.Fatalf("Yaml index %q failed to parse: %s", testfile, err)
}
verifyLocalIndex(t, yamlIndex)
// Check that charts file is also created
chartsFile := filepath.Join(r.CachePath, helmpath.CacheChartsFile(r.Config.Name))
if _, err := os.Stat(chartsFile); err != nil {
t.Fatalf("error finding created charts file: %#v", err)
}
chartsFileBytes, err := ioutil.ReadFile(chartsFile)
if err != nil {
t.Fatalf("error reading charts file: %#v", err)
}
verifyLocalChartsFile(t, chartsFileBytes, yamlIndex)
})
t.Run("should not decode the path in the repo url while downloading index", func(t *testing.T) {

36
pkg/repo/repotest/server.go
Unescape View File

@ -81,6 +81,7 @@ type Server struct {
srv *httptest.Server
middleware http.HandlerFunc
jsonIndexEnabled bool
yamlIndexDisabled bool
}
// WithMiddleware injects middleware in front of the server. This can be used to inject
@ -96,9 +97,16 @@ func (s *Server) Root() string {
// ServeJSONIndex allows you to enable or disable serving
// index.json repository index file
func (s *Server) ServeJSONIndex(enabled bool) {
func (s *Server) ServeJSONIndex(enabled bool) error {
s.jsonIndexEnabled = enabled
s.CreateIndex()
return s.CreateIndex()
}
// ServeYamlIndex allows you to enable or disable serving
// index.yaml repository index file
func (s *Server) ServeYamlIndex(enabled bool) error {
s.yamlIndexDisabled = !enabled
return s.CreateIndex()
}
// CopyCharts takes a glob expression and copies those charts to the server root.
@ -127,26 +135,34 @@ func (s *Server) CopyCharts(origin string) ([]string, error) {
// CreateIndex will read docroot and generate an index.yaml file.
func (s *Server) CreateIndex() error {
// generate the index
index, err := repo.IndexDirectory(s.docroot, s.URL())
yamlIndexFile := filepath.Join(s.docroot, "index.yaml")
jsonIndexFile := filepath.Join(s.docroot, "index.json")
// cleanup existing files
err := os.RemoveAll(yamlIndexFile)
if err != nil {
return err
}
err = os.RemoveAll(jsonIndexFile)
if err != nil {
return err
}
yamlIndexFile := filepath.Join(s.docroot, "index.yaml")
if !s.jsonIndexEnabled && s.yamlIndexDisabled {
return nil
}
err = index.WriteFile(yamlIndexFile, 0644)
// generate the index
index, err := repo.IndexDirectory(s.docroot, s.URL())
if err != nil {
return err
}
jsonIndexFile := filepath.Join(s.docroot, "index.json")
// cleanup existing files
err = os.RemoveAll(jsonIndexFile)
if !s.yamlIndexDisabled {
err = index.WriteFile(yamlIndexFile, 0644)
if err != nil {
return err
}
}
if s.jsonIndexEnabled {
err = index.WriteJSONFile(jsonIndexFile, 0644)

68
pkg/repo/repotest/server_test.go
Unescape View File

@ -112,7 +112,10 @@ func TestServer(t *testing.T) {
}
// Let's enable the toggle to serve index.json
srv.ServeJSONIndex(true)
err = srv.ServeJSONIndex(true)
if err != nil {
t.Fatal(err)
}
// Now we should be able to fetch the index.json
// with appropriate content
@ -146,7 +149,10 @@ func TestServer(t *testing.T) {
}
// Let's disable the toggle to serve index.json
srv.ServeJSONIndex(false)
err = srv.ServeJSONIndex(false)
if err != nil {
t.Fatal(err)
}
// Now the below should give 404
res, err = http.Get(srv.URL() + "/index.json")
@ -159,6 +165,64 @@ func TestServer(t *testing.T) {
}
}
func TestServer_ServeYamlIndex(t *testing.T) {
defer ensure.HelmHome(t)()
rootDir := ensure.TempDir(t)
srv := NewServer(rootDir)
defer srv.Stop()
_, err := srv.CopyCharts("testdata/*.tgz")
if err != nil {
// Some versions of Go don't correctly fire defer on Fatal.
t.Fatal(err)
}
// By default index.yaml should be served
res, err := http.Get(srv.URL() + "/index.yaml")
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 200 {
t.Fatalf("Expected 200, got %d", res.StatusCode)
}
// By default index.yaml should be served, yes,
// unless a toggle is disabled.
// Let's disable the toggle to serve index.json
err = srv.ServeYamlIndex(false)
if err != nil {
t.Fatal(err)
}
// So the below should give 404
res, err = http.Get(srv.URL() + "/index.yaml")
res.Body.Close()
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 404 {
t.Fatalf("Expected 404, got %d", res.StatusCode)
}
// Let's enable the toggle to serve index.yaml
err = srv.ServeYamlIndex(true)
if err != nil {
t.Fatal(err)
}
// Now the below should give 200
res, err = http.Get(srv.URL() + "/index.yaml")
res.Body.Close()
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 200 {
t.Fatalf("Expected 200, got %d", res.StatusCode)
}
}
func TestNewTempServer(t *testing.T) {
defer ensure.HelmHome(t)()

Write Preview
Loading…
Cancel
Save