diff --git a/cmd/helm/repo_add_test.go b/cmd/helm/repo_add_test.go index d1b9bc0fc..d791dbd46 100644 --- a/cmd/helm/repo_add_test.go +++ b/cmd/helm/repo_add_test.go @@ -104,6 +104,67 @@ func TestRepoAdd(t *testing.T) { } } +func TestRepoAdd_IndexJSON(t *testing.T) { + ts, err := repotest.NewTempServer("testdata/testserver/*.*") + if err != nil { + t.Fatal(err) + } + err = ts.ServeJSONIndex(true) + if err != nil { + t.Fatal(err) + } + err = ts.ServeYamlIndex(false) + if err != nil { + t.Fatal(err) + } + defer ts.Stop() + + rootDir := ensure.TempDir(t) + repoFile := filepath.Join(rootDir, "repositories.yaml") + + const testRepoName = "test-name" + + o := &repoAddOptions{ + name: testRepoName, + url: ts.URL(), + noUpdate: true, + repoFile: repoFile, + } + os.Setenv(xdg.CacheHomeEnvVar, rootDir) + + if err := o.run(ioutil.Discard); err != nil { + t.Error(err) + } + + f, err := repo.LoadFile(repoFile) + if err != nil { + t.Fatal(err) + } + + if !f.Has(testRepoName) { + t.Errorf("%s was not successfully inserted into %s", testRepoName, repoFile) + } + + idx := filepath.Join(helmpath.CachePath("repository"), helmpath.CacheIndexJSONFile(testRepoName)) + if _, err := os.Stat(idx); os.IsNotExist(err) { + t.Errorf("Error cache json index file was not created for repository %s", testRepoName) + } + idx = filepath.Join(helmpath.CachePath("repository"), helmpath.CacheChartsFile(testRepoName)) + if _, err := os.Stat(idx); os.IsNotExist(err) { + t.Errorf("Error cache charts file was not created for repository %s", testRepoName) + } + + o.noUpdate = false + + if err := o.run(ioutil.Discard); err != nil { + t.Errorf("Repository was not updated: %s", err) + } + + if err := o.run(ioutil.Discard); err != nil { + t.Errorf("Duplicate repository name was added") + } +} + func TestRepoAddConcurrentGoRoutines(t *testing.T) { const testName = "test-name" repoFile := filepath.Join(ensure.TempDir(t), "repositories.yaml") diff --git a/pkg/helmpath/home.go b/pkg/helmpath/home.go index bd43e8890..993a70798 100644 --- a/pkg/helmpath/home.go +++ b/pkg/helmpath/home.go @@ -34,6 +34,14 @@ func CacheIndexFile(name string) string { return name + "index.yaml" } +// CacheIndexJSONFile returns the path to an json index for the given named repository. +func CacheIndexJSONFile(name string) string { + if name != "" { + name += "-" + } + return name + "index.json" +} + // CacheChartsFile returns the path to a text file listing all the charts // within the given named repository. func CacheChartsFile(name string) string { diff --git a/pkg/repo/chartrepo.go b/pkg/repo/chartrepo.go index c2c366a1e..040158783 100644 --- a/pkg/repo/chartrepo.go +++ b/pkg/repo/chartrepo.go @@ -113,12 +113,66 @@ func (r *ChartRepository) Load() error { // DownloadIndexFile fetches the index from a repository. func (r *ChartRepository) DownloadIndexFile() (string, error) { - parsedURL, err := url.Parse(r.Config.URL) + indexFile, err := r.downloadJSONOrYamlIndex() if err != nil { return "", err } - parsedURL.RawPath = path.Join(parsedURL.RawPath, "index.yaml") - parsedURL.Path = path.Join(parsedURL.Path, "index.yaml") + + // Create the chart list file in the cache directory + var charts strings.Builder + for name := range indexFile.Entries { + fmt.Fprintln(&charts, name) + } + chartsFile := filepath.Join(r.CachePath, helmpath.CacheChartsFile(r.Config.Name)) + os.MkdirAll(filepath.Dir(chartsFile), 0755) + ioutil.WriteFile(chartsFile, []byte(charts.String()), 0644) + + // Create the yaml index file in the cache directory + fname := filepath.Join(r.CachePath, helmpath.CacheIndexFile(r.Config.Name)) + os.MkdirAll(filepath.Dir(fname), 0755) + + // Create the json index file in the cache directory + jsonIndexFilePapth := filepath.Join(r.CachePath, helmpath.CacheIndexJSONFile(r.Config.Name)) + os.MkdirAll(filepath.Dir(jsonIndexFilePapth), 0755) + + _ = indexFile.WriteJSONFile(jsonIndexFilePapth, 0644) + + return fname, indexFile.WriteFile(fname, 0644) +} + +func (r *ChartRepository) downloadJSONOrYamlIndex() (*IndexFile, error) { + jsonIndexBytes, err := r.downloadIndexFile("index.json") + if err == nil { + // if json index has been downloaded without errors, + // all good, use it! + indexFile, err := loadIndexJSON(jsonIndexBytes) + if err == nil { + return indexFile, nil + } + } + + // if json index download or parse has issues, + // swallow errors and fallback to using yaml index + yamlIndexBytes, err := r.downloadIndexFile("index.yaml") + if err != nil { + return nil, err + } + + indexFile, err := loadIndex(yamlIndexBytes) + if err != nil { + return nil, err + } + + return indexFile, nil +} + +func (r *ChartRepository) downloadIndexFile(indexFileName string) ([]byte, error) { + parsedURL, err := url.Parse(r.Config.URL) + if err != nil { + return nil, err + } + parsedURL.RawPath = path.Join(parsedURL.RawPath, indexFileName) + parsedURL.Path = path.Join(parsedURL.Path, indexFileName) indexURL := parsedURL.String() // TODO add user-agent @@ -129,32 +183,14 @@ func (r *ChartRepository) DownloadIndexFile() (string, error) { getter.WithBasicAuth(r.Config.Username, r.Config.Password), ) if err != nil { - return "", err - } - - index, err := ioutil.ReadAll(resp) - if err != nil { - return "", err + return nil, err } - indexFile, err := loadIndex(index) + indexBytes, err := ioutil.ReadAll(resp) if err != nil { - return "", err + return nil, err } - - // Create the chart list file in the cache directory - var charts strings.Builder - for name := range indexFile.Entries { - fmt.Fprintln(&charts, name) - } - chartsFile := filepath.Join(r.CachePath, helmpath.CacheChartsFile(r.Config.Name)) - os.MkdirAll(filepath.Dir(chartsFile), 0755) - ioutil.WriteFile(chartsFile, []byte(charts.String()), 0644) - - // Create the index file in the cache directory - fname := filepath.Join(r.CachePath, helmpath.CacheIndexFile(r.Config.Name)) - os.MkdirAll(filepath.Dir(fname), 0755) - return fname, ioutil.WriteFile(fname, index, 0644) + return indexBytes, nil } // Index generates an index for the chart repository and writes an index.yaml file. diff --git a/pkg/repo/index_test.go b/pkg/repo/index_test.go index 6e03b8e60..a3ed0392a 100644 --- a/pkg/repo/index_test.go +++ b/pkg/repo/index_test.go @@ -170,8 +170,17 @@ func TestMerge(t *testing.T) { } func TestDownloadIndexFile(t *testing.T) { - t.Run("should download index file", func(t *testing.T) { - srv, err := startLocalServerForTests(nil) + t.Run("should download index.yaml file and create index.yaml and index.json", func(t *testing.T) { + fileBytes, err := ioutil.ReadFile("testdata/local-index.yaml") + if err != nil { + t.Fatal(err) + } + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/index.yaml" { + w.Write(fileBytes) + } + }) + srv, err := startLocalServerForTests(handler) if err != nil { t.Fatal(err) } @@ -185,37 +194,126 @@ func TestDownloadIndexFile(t *testing.T) { t.Errorf("Problem creating chart repository from %s: %v", testRepo, err) } - idx, err := r.DownloadIndexFile() + yamlIndexFilePath, err := r.DownloadIndexFile() if err != nil { - t.Fatalf("Failed to download index file to %s: %#v", idx, err) + t.Fatalf("Failed to download index file: %#v", err) } - if _, err := os.Stat(idx); err != nil { + if _, err := os.Stat(yamlIndexFilePath); err != nil { t.Fatalf("error finding created index file: %#v", err) } - b, err := ioutil.ReadFile(idx) + yamlIndexBytes, err := ioutil.ReadFile(yamlIndexFilePath) if err != nil { - t.Fatalf("error reading index file: %#v", err) + t.Fatalf("error reading yaml index file: %#v", err) } - i, err := loadIndex(b) + yamlIndex, err := loadIndex(yamlIndexBytes) if err != nil { - t.Fatalf("Index %q failed to parse: %s", testfile, err) + t.Fatalf("Yaml index %q failed to parse: %s", testfile, err) } - verifyLocalIndex(t, i) + verifyLocalIndex(t, yamlIndex) + + jsonIndexFilePath := filepath.Join(helmpath.CachePath("repository"), helmpath.CacheIndexJSONFile(testRepo)) + if _, err := os.Stat(jsonIndexFilePath); err != nil { + t.Fatalf("error finding created json index file: %#v", err) + } + + jsonBytes, err := ioutil.ReadFile(jsonIndexFilePath) + if err != nil { + t.Fatalf("error reading json index file: %#v", err) + } + + jsonIndex, err := loadIndexJSON(jsonBytes) + if err != nil { + t.Fatalf("Json index %q failed to parse: %s", testjsonfile, err) + } + verifyLocalIndex(t, jsonIndex) // Check that charts file is also created - idx = filepath.Join(r.CachePath, helmpath.CacheChartsFile(r.Config.Name)) - if _, err := os.Stat(idx); err != nil { + chartsFile := filepath.Join(r.CachePath, helmpath.CacheChartsFile(r.Config.Name)) + if _, err := os.Stat(chartsFile); err != nil { t.Fatalf("error finding created charts file: %#v", err) } - b, err = ioutil.ReadFile(idx) + chartsFileBytes, err := ioutil.ReadFile(chartsFile) if err != nil { t.Fatalf("error reading charts file: %#v", err) } - verifyLocalChartsFile(t, b, i) + verifyLocalChartsFile(t, chartsFileBytes, yamlIndex) + }) + + t.Run("should download index.json file and create index.json and index.yaml", func(t *testing.T) { + fileBytes, err := ioutil.ReadFile("testdata/local-index.json") + if err != nil { + t.Fatal(err) + } + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/index.json" { + w.Write(fileBytes) + } + }) + srv, err := startLocalServerForTests(handler) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + + r, err := NewChartRepository(&Entry{ + Name: testRepo, + URL: srv.URL, + }, getter.All(&cli.EnvSettings{})) + if err != nil { + t.Errorf("Problem creating chart repository from %s: %v", testRepo, err) + } + + yamlIndexFilePath, err := r.DownloadIndexFile() + if err != nil { + t.Fatalf("Failed to download index file: %#v", err) + } + + jsonIndexFilePath := filepath.Join(helmpath.CachePath("repository"), helmpath.CacheIndexJSONFile(testRepo)) + if _, err := os.Stat(jsonIndexFilePath); err != nil { + t.Fatalf("error finding created json index file: %#v", err) + } + + jsonBytes, err := ioutil.ReadFile(jsonIndexFilePath) + if err != nil { + t.Fatalf("error reading json index file: %#v", err) + } + + jsonIndex, err := loadIndexJSON(jsonBytes) + if err != nil { + t.Fatalf("Json index %q failed to parse: %s", testjsonfile, err) + } + verifyLocalIndex(t, jsonIndex) + + if _, err := os.Stat(yamlIndexFilePath); err != nil { + t.Fatalf("error finding created yaml index file: %#v", err) + } + + yamlIndexBytes, err := ioutil.ReadFile(yamlIndexFilePath) + if err != nil { + t.Fatalf("error reading yaml index file: %#v", err) + } + + yamlIndex, err := loadIndex(yamlIndexBytes) + if err != nil { + t.Fatalf("Yaml index %q failed to parse: %s", testfile, err) + } + verifyLocalIndex(t, yamlIndex) + + // Check that charts file is also created + chartsFile := filepath.Join(r.CachePath, helmpath.CacheChartsFile(r.Config.Name)) + if _, err := os.Stat(chartsFile); err != nil { + t.Fatalf("error finding created charts file: %#v", err) + } + + chartsFileBytes, err := ioutil.ReadFile(chartsFile) + if err != nil { + t.Fatalf("error reading charts file: %#v", err) + } + verifyLocalChartsFile(t, chartsFileBytes, yamlIndex) }) t.Run("should not decode the path in the repo url while downloading index", func(t *testing.T) { diff --git a/pkg/repo/repotest/server.go b/pkg/repo/repotest/server.go index c1123edaa..f6e5cee73 100644 --- a/pkg/repo/repotest/server.go +++ b/pkg/repo/repotest/server.go @@ -77,10 +77,11 @@ func NewServer(docroot string) *Server { // Server is an implementation of a repository server for testing. type Server struct { - docroot string - srv *httptest.Server - middleware http.HandlerFunc - jsonIndexEnabled bool + docroot string + srv *httptest.Server + middleware http.HandlerFunc + jsonIndexEnabled bool + yamlIndexDisabled bool } // WithMiddleware injects middleware in front of the server. This can be used to inject @@ -96,9 +97,16 @@ func (s *Server) Root() string { // ServeJSONIndex allows you to enable or disable serving // index.json repository index file -func (s *Server) ServeJSONIndex(enabled bool) { +func (s *Server) ServeJSONIndex(enabled bool) error { s.jsonIndexEnabled = enabled - s.CreateIndex() + return s.CreateIndex() +} + +// ServeYamlIndex allows you to enable or disable serving +// index.yaml repository index file +func (s *Server) ServeYamlIndex(enabled bool) error { + s.yamlIndexDisabled = !enabled + return s.CreateIndex() } // CopyCharts takes a glob expression and copies those charts to the server root. @@ -127,27 +135,35 @@ func (s *Server) CopyCharts(origin string) ([]string, error) { // CreateIndex will read docroot and generate an index.yaml file. func (s *Server) CreateIndex() error { - - // generate the index - index, err := repo.IndexDirectory(s.docroot, s.URL()) + yamlIndexFile := filepath.Join(s.docroot, "index.yaml") + jsonIndexFile := filepath.Join(s.docroot, "index.json") + // cleanup existing files + err := os.RemoveAll(yamlIndexFile) if err != nil { return err } - - yamlIndexFile := filepath.Join(s.docroot, "index.yaml") - - err = index.WriteFile(yamlIndexFile, 0644) + err = os.RemoveAll(jsonIndexFile) if err != nil { return err } - jsonIndexFile := filepath.Join(s.docroot, "index.json") - // cleanup existing files - err = os.RemoveAll(jsonIndexFile) + if !s.jsonIndexEnabled && s.yamlIndexDisabled { + return nil + } + + // generate the index + index, err := repo.IndexDirectory(s.docroot, s.URL()) if err != nil { return err } + if !s.yamlIndexDisabled { + err = index.WriteFile(yamlIndexFile, 0644) + if err != nil { + return err + } + } + if s.jsonIndexEnabled { err = index.WriteJSONFile(jsonIndexFile, 0644) if err != nil { diff --git a/pkg/repo/repotest/server_test.go b/pkg/repo/repotest/server_test.go index 46c704638..508574597 100644 --- a/pkg/repo/repotest/server_test.go +++ b/pkg/repo/repotest/server_test.go @@ -112,7 +112,10 @@ func TestServer(t *testing.T) { } // Let's enable the toggle to serve index.json - srv.ServeJSONIndex(true) + err = srv.ServeJSONIndex(true) + if err != nil { + t.Fatal(err) + } // Now we should be able to fetch the index.json // with appropriate content @@ -146,7 +149,10 @@ func TestServer(t *testing.T) { } // Let's disable the toggle to serve index.json - srv.ServeJSONIndex(false) + err = srv.ServeJSONIndex(false) + if err != nil { + t.Fatal(err) + } // Now the below should give 404 res, err = http.Get(srv.URL() + "/index.json") @@ -159,6 +165,64 @@ func TestServer(t *testing.T) { } } +func TestServer_ServeYamlIndex(t *testing.T) { + defer ensure.HelmHome(t)() + + rootDir := ensure.TempDir(t) + + srv := NewServer(rootDir) + defer srv.Stop() + + _, err := srv.CopyCharts("testdata/*.tgz") + if err != nil { + // Some versions of Go don't correctly fire defer on Fatal. + t.Fatal(err) + } + + // By default index.yaml should be served + res, err := http.Get(srv.URL() + "/index.yaml") + if err != nil { + t.Fatal(err) + } + if res.StatusCode != 200 { + t.Fatalf("Expected 200, got %d", res.StatusCode) + } + + // By default index.yaml should be served, yes, + // unless a toggle is disabled. + // Let's disable the toggle to serve index.json + err = srv.ServeYamlIndex(false) + if err != nil { + t.Fatal(err) + } + + // So the below should give 404 + res, err = http.Get(srv.URL() + "/index.yaml") + res.Body.Close() + if err != nil { + t.Fatal(err) + } + if res.StatusCode != 404 { + t.Fatalf("Expected 404, got %d", res.StatusCode) + } + + // Let's enable the toggle to serve index.yaml + err = srv.ServeYamlIndex(true) + if err != nil { + t.Fatal(err) + } + + // Now the below should give 200 + res, err = http.Get(srv.URL() + "/index.yaml") + res.Body.Close() + if err != nil { + t.Fatal(err) + } + if res.StatusCode != 200 { + t.Fatalf("Expected 200, got %d", res.StatusCode) + } +} + func TestNewTempServer(t *testing.T) { defer ensure.HelmHome(t)()