From 99009c7be85ca7337245bfa5d93162bc7b2a61a0 Mon Sep 17 00:00:00 2001 From: MichaelMorris Date: Thu, 19 Sep 2024 15:54:15 +0100 Subject: [PATCH] Reapply "Improve helm dependency update performance" This reverts commit c81bd8912e67177f7941714098e762a88f04c430. Signed-off-by: MichaelMorris --- internal/resolver/resolver.go | 43 +++++++++++++++++++++--------- internal/resolver/resolver_test.go | 2 +- pkg/downloader/manager.go | 30 ++++++++++++++++----- pkg/downloader/manager_test.go | 8 +++--- 4 files changed, 58 insertions(+), 25 deletions(-) diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 13dcd2ce9..554d645b0 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -53,21 +53,23 @@ func New(chartpath, cachepath string, registryClient *registry.Client) *Resolver } // Resolve resolves dependencies and returns a lock file with the resolution. -func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string) (*chart.Lock, error) { +func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string) (*chart.Lock, map[string]string, error) { // Now we clone the dependencies, locking as we go. locked := make([]*chart.Dependency, len(reqs)) missing := []string{} + loadedIndexFiles := make(map[string]*repo.IndexFile) + urls := make(map[string]string) for i, d := range reqs { constraint, err := semver.NewConstraint(d.Version) if err != nil { - return nil, fmt.Errorf("dependency %q has an invalid version/constraint format: %w", d.Name, err) + return nil, nil, fmt.Errorf("dependency %q has an invalid version/constraint format: %w", d.Name, err) } if d.Repository == "" { // Local chart subfolder if _, err := GetLocalPath(filepath.Join("charts", d.Name), r.chartpath); err != nil { - return nil, err + return nil, nil, err } locked[i] = &chart.Dependency{ @@ -80,12 +82,12 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string if strings.HasPrefix(d.Repository, "file://") { chartpath, err := GetLocalPath(d.Repository, r.chartpath) if err != nil { - return nil, err + return nil, nil, err } ch, err := loader.LoadDir(chartpath) if err != nil { - return nil, err + return nil, nil, err } v, err := semver.NewVersion(ch.Metadata.Version) @@ -123,14 +125,26 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string var ok bool found := true if !registry.IsOCI(d.Repository) { - repoIndex, err := repo.LoadIndexFile(filepath.Join(r.cachepath, helmpath.CacheIndexFile(repoName))) - if err != nil { - return nil, fmt.Errorf("no cached repository for %s found. (try 'helm repo update'): %w", repoName, err) + filepath := filepath.Join(r.cachepath, helmpath.CacheIndexFile(repoName)) + var repoIndex *repo.IndexFile + + // Store previously loaded index files in a map. If repositories share the + // same index file there is no need to reload the same file again. This + // improves performance. + if indexFile, loaded := loadedIndexFiles[filepath]; !loaded { + var err error + repoIndex, err = repo.LoadIndexFile(filepath) + loadedIndexFiles[filepath] = repoIndex + if err != nil { + return nil, nil, fmt.Errorf("no cached repository for %s found. (try 'helm repo update'): %w", repoName, err) + } + } else { + repoIndex = indexFile } vs, ok = repoIndex.Entries[d.Name] if !ok { - return nil, fmt.Errorf("%s chart not found in repo %s", d.Name, d.Repository) + return nil, nil, fmt.Errorf("%s chart not found in repo %s", d.Name, d.Repository) } found = false } else { @@ -152,7 +166,7 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string ref := fmt.Sprintf("%s/%s", strings.TrimPrefix(d.Repository, fmt.Sprintf("%s://", registry.OCIScheme)), d.Name) tags, err := r.registryClient.Tags(ref) if err != nil { - return nil, fmt.Errorf("could not retrieve list of tags for repository %s: %w", d.Repository, err) + return nil, nil, fmt.Errorf("could not retrieve list of tags for repository %s: %w", d.Repository, err) } vs = make(repo.ChartVersions, len(tags)) @@ -183,6 +197,9 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string } if constraint.Check(v) { found = true + if len(ver.URLs) > 0 { + urls[d.Repository+ver.Name+ver.Version] = ver.URLs[0] + } locked[i].Version = v.Original() break } @@ -193,19 +210,19 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string } } if len(missing) > 0 { - return nil, fmt.Errorf("can't get a valid version for %d subchart(s): %s. Make sure a matching chart version exists in the repo, or change the version constraint in Chart.yaml", len(missing), strings.Join(missing, ", ")) + return nil, nil, fmt.Errorf("can't get a valid version for %d subchart(s): %s. Make sure a matching chart version exists in the repo, or change the version constraint in Chart.yaml", len(missing), strings.Join(missing, ", ")) } digest, err := HashReq(reqs, locked) if err != nil { - return nil, err + return nil, nil, err } return &chart.Lock{ Generated: time.Now(), Digest: digest, Dependencies: locked, - }, nil + }, urls, nil } // HashReq generates a hash of the dependencies. diff --git a/internal/resolver/resolver_test.go b/internal/resolver/resolver_test.go index 1e33837a9..766443a7c 100644 --- a/internal/resolver/resolver_test.go +++ b/internal/resolver/resolver_test.go @@ -144,7 +144,7 @@ func TestResolve(t *testing.T) { r := New("testdata/chartpath", "testdata/repository", registryClient) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - l, err := r.Resolve(tt.req, repoNames) + l, _, err := r.Resolve(tt.req, repoNames) if err != nil { if tt.err { return diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index b43165975..3eb6e72d9 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -141,7 +141,7 @@ func (m *Manager) Build() error { } // Now we need to fetch every package here into charts/ - return m.downloadAll(lock.Dependencies) + return m.downloadAll(lock.Dependencies, nil) } // Update updates a local charts directory. @@ -191,13 +191,13 @@ func (m *Manager) Update() error { // Now we need to find out which version of a chart best satisfies the // dependencies in the Chart.yaml - lock, err := m.resolve(req, repoNames) + lock, urls, err := m.resolve(req, repoNames) if err != nil { return err } // Now we need to fetch every package here into charts/ - if err := m.downloadAll(lock.Dependencies); err != nil { + if err := m.downloadAll(lock.Dependencies, urls); err != nil { return err } @@ -230,7 +230,7 @@ func (m *Manager) loadChartDir() (*chart.Chart, error) { // resolve takes a list of dependencies and translates them into an exact version to download. // // This returns a lock file, which has all of the dependencies normalized to a specific version. -func (m *Manager) resolve(req []*chart.Dependency, repoNames map[string]string) (*chart.Lock, error) { +func (m *Manager) resolve(req []*chart.Dependency, repoNames map[string]string) (*chart.Lock, map[string]string, error) { res := resolver.New(m.ChartPath, m.RepositoryCache, m.RegistryClient) return res.Resolve(req, repoNames) } @@ -239,7 +239,7 @@ func (m *Manager) resolve(req []*chart.Dependency, repoNames map[string]string) // // It will delete versions of the chart that exist on disk and might cause // a conflict. -func (m *Manager) downloadAll(deps []*chart.Dependency) error { +func (m *Manager) downloadAll(deps []*chart.Dependency, urls map[string]string) error { repos, err := m.loadChartRepositories() if err != nil { return err @@ -312,7 +312,7 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { // Any failure to resolve/download a chart should fail: // https://github.com/helm/helm/issues/1439 - churl, username, password, insecureskiptlsverify, passcredentialsall, caFile, certFile, keyFile, err := m.findChartURL(dep.Name, dep.Version, dep.Repository, repos) + churl, username, password, insecureskiptlsverify, passcredentialsall, caFile, certFile, keyFile, err := m.findChartURL(dep.Name, dep.Version, dep.Repository, repos, urls) if err != nil { saveError = fmt.Errorf("could not find %s: %w", churl, err) break @@ -501,6 +501,7 @@ func (m *Manager) ensureMissingRepos(repoNames map[string]string, deps []*chart. var ru []*repo.Entry +Outer: for _, dd := range deps { // If the chart is in the local charts directory no repository needs @@ -528,6 +529,14 @@ func (m *Manager) ensureMissingRepos(repoNames map[string]string, deps []*chart. repoNames[dd.Name] = rn + // If repository is already present don't add to array. This will skip + // unnecessary index file downloading improving performance. + for _, item := range ru { + if item.URL == dd.Repository { + continue Outer + } + } + // Assuming the repository is generally available. For Helm managed // access controls the repository needs to be added through the user // managed system. This path will work for public charts, like those @@ -721,7 +730,7 @@ func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error { // repoURL is the repository to search // // If it finds a URL that is "relative", it will prepend the repoURL. -func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (url, username, password string, insecureskiptlsverify, passcredentialsall bool, caFile, certFile, keyFile string, err error) { +func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository, urls map[string]string) (url, username, password string, insecureskiptlsverify, passcredentialsall bool, caFile, certFile, keyFile string, err error) { if registry.IsOCI(repoURL) { return fmt.Sprintf("%s/%s:%s", repoURL, name, version), "", "", false, false, "", "", "", nil } @@ -759,7 +768,14 @@ func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]* return } } + + urlsKey := repoURL + name + version + if _, ok := urls[urlsKey]; ok { + url = urls[urlsKey] + } else { url, err = repo.FindChartInRepoURL(repoURL, name, m.Getters, repo.WithChartVersion(version), repo.WithClientTLS(certFile, keyFile, caFile)) + } + if err == nil { return url, username, password, false, false, "", "", "", err } diff --git a/pkg/downloader/manager_test.go b/pkg/downloader/manager_test.go index f01a5d7ad..f3df04f51 100644 --- a/pkg/downloader/manager_test.go +++ b/pkg/downloader/manager_test.go @@ -71,7 +71,7 @@ func TestFindChartURL(t *testing.T) { version := "0.1.0" repoURL := "http://example.com/charts" - churl, username, password, insecureSkipTLSVerify, passcredentialsall, _, _, _, err := m.findChartURL(name, version, repoURL, repos) + churl, username, password, insecureSkipTLSVerify, passcredentialsall, _, _, _, err := m.findChartURL(name, version, repoURL, repos, make(map[string]string)) if err != nil { t.Fatal(err) } @@ -96,7 +96,7 @@ func TestFindChartURL(t *testing.T) { version = "1.2.3" repoURL = "https://example-https-insecureskiptlsverify.com" - churl, username, password, insecureSkipTLSVerify, passcredentialsall, _, _, _, err = m.findChartURL(name, version, repoURL, repos) + churl, username, password, insecureSkipTLSVerify, passcredentialsall, _, _, _, err = m.findChartURL(name, version, repoURL, repos, make(map[string]string)) if err != nil { t.Fatal(err) } @@ -267,7 +267,7 @@ func TestDownloadAll(t *testing.T) { if err := os.MkdirAll(filepath.Join(chartPath, "tmpcharts"), 0755); err != nil { t.Fatal(err) } - if err := m.downloadAll([]*chart.Dependency{signDep, localDep}); err != nil { + if err := m.downloadAll([]*chart.Dependency{signDep, localDep}, make(map[string]string)); err != nil { t.Error(err) } @@ -296,7 +296,7 @@ version: 0.1.0` Version: "0.1.0", } - err = m.downloadAll([]*chart.Dependency{badLocalDep}) + err = m.downloadAll([]*chart.Dependency{badLocalDep}, make(map[string]string)) if err == nil { t.Fatal("Expected error for bad dependency name") }