From 0cf7b19b216ccc17314dccb9a0d1322f4bfe1983 Mon Sep 17 00:00:00 2001 From: Suleiman Dibirov Date: Sat, 10 Aug 2024 08:20:24 +0300 Subject: [PATCH 1/5] feat(dependency): Add --skip-download-if-exists into dependency build/update Signed-off-by: Suleiman Dibirov --- pkg/action/dependency.go | 1 + pkg/cmd/dependency.go | 1 + pkg/cmd/dependency_build.go | 1 + pkg/cmd/dependency_update.go | 1 + pkg/downloader/manager.go | 10 ++++++++++ 5 files changed, 14 insertions(+) diff --git a/pkg/action/dependency.go b/pkg/action/dependency.go index 03c370c8e..82b69b676 100644 --- a/pkg/action/dependency.go +++ b/pkg/action/dependency.go @@ -37,6 +37,7 @@ type Dependency struct { Verify bool Keyring string SkipRefresh bool + SkipDownloadIfExists bool ColumnWidth uint Username string Password string diff --git a/pkg/cmd/dependency.go b/pkg/cmd/dependency.go index 34bbff6be..587b16dc7 100644 --- a/pkg/cmd/dependency.go +++ b/pkg/cmd/dependency.go @@ -133,4 +133,5 @@ func addDependencySubcommandFlags(f *pflag.FlagSet, client *action.Dependency) { f.BoolVar(&client.InsecureSkipTLSverify, "insecure-skip-tls-verify", false, "skip tls certificate checks for the chart download") f.BoolVar(&client.PlainHTTP, "plain-http", false, "use insecure HTTP connections for the chart download") f.StringVar(&client.CaFile, "ca-file", "", "verify certificates of HTTPS-enabled servers using this CA bundle") + f.BoolVar(&client.SkipDownloadIfExists, "skip-download-if-exists", false, "skip download of the chart if it already exists in the charts directory") } diff --git a/pkg/cmd/dependency_build.go b/pkg/cmd/dependency_build.go index 320fe12ae..08075845e 100644 --- a/pkg/cmd/dependency_build.go +++ b/pkg/cmd/dependency_build.go @@ -65,6 +65,7 @@ func newDependencyBuildCmd(out io.Writer) *cobra.Command { ChartPath: chartpath, Keyring: client.Keyring, SkipUpdate: client.SkipRefresh, + SkipDownloadIfExists: client.SkipDownloadIfExists, Getters: getter.All(settings), RegistryClient: registryClient, RepositoryConfig: settings.RepositoryConfig, diff --git a/pkg/cmd/dependency_update.go b/pkg/cmd/dependency_update.go index b534fb48a..714893ea1 100644 --- a/pkg/cmd/dependency_update.go +++ b/pkg/cmd/dependency_update.go @@ -69,6 +69,7 @@ func newDependencyUpdateCmd(_ *action.Configuration, out io.Writer) *cobra.Comma ChartPath: chartpath, Keyring: client.Keyring, SkipUpdate: client.SkipRefresh, + SkipDownloadIfExists: client.SkipDownloadIfExists, Getters: getter.All(settings), RegistryClient: registryClient, RepositoryConfig: settings.RepositoryConfig, diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index d41b8fdb4..3bb776473 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -70,6 +70,8 @@ type Manager struct { Keyring string // SkipUpdate indicates that the repository should not be updated first. SkipUpdate bool + // SkipDownloadIfExists indicates that the chart should not be downloaded if it already exists. + SkipDownloadIfExists bool // Getter collection for the operation Getters []getter.Provider RegistryClient *registry.Client @@ -326,6 +328,14 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { continue } + if m.SkipDownloadIfExists { + name := filepath.Base(churl) + if _, err = os.Stat(filepath.Join(destPath, name)); err == nil { + fmt.Fprintf(m.Out, "Already exists locally %s from repo %s\n", dep.Name, dep.Repository) + continue + } + } + fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) dl := ChartDownloader{ From f1d5437377a8111e078b67d3ab98ba6340bd0dbe Mon Sep 17 00:00:00 2001 From: Suleiman Dibirov Date: Wed, 4 Sep 2024 17:27:31 +0300 Subject: [PATCH 2/5] added oci support Signed-off-by: Suleiman Dibirov --- pkg/downloader/chart_downloader.go | 71 ++++++++++++++++++++++++++---- pkg/downloader/manager.go | 28 ++++++++---- 2 files changed, 83 insertions(+), 16 deletions(-) diff --git a/pkg/downloader/chart_downloader.go b/pkg/downloader/chart_downloader.go index 00c8c56e8..fc70c5914 100644 --- a/pkg/downloader/chart_downloader.go +++ b/pkg/downloader/chart_downloader.go @@ -149,12 +149,7 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven } } - name := filepath.Base(u.Path) - if u.Scheme == registry.OCIScheme { - idx := strings.LastIndexByte(name, ':') - name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) - } - + name := c.getChartName(u.String()) destfile := filepath.Join(dest, name) if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil { return destfile, nil, err @@ -353,11 +348,12 @@ func (c *ChartDownloader) DownloadToCache(ref, version string) (string, *provena // // TODO: support OCI hash func (c *ChartDownloader) ResolveChartVersion(ref, version string) (string, *url.URL, error) { - u, err := url.Parse(ref) + u, err := c.parseChartURL(ref, version) if err != nil { - return "", nil, fmt.Errorf("invalid chart URL format: %s", ref) + return "", u, err } + if registry.IsOCI(u.String()) { if c.RegistryClient == nil { return "", nil, fmt.Errorf("unable to lookup ref %s at version '%s', missing registry client", ref, version) @@ -574,6 +570,65 @@ func (c *ChartDownloader) scanReposForURL(u string, rf *repo.File) (*repo.Entry, return nil, ErrNoOwnerRepo } +func (c *ChartDownloader) getChartName(url string) string { + name := filepath.Base(url) + if registry.IsOCI(url) { + idx := strings.LastIndexByte(name, ':') + name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) + } + + return name +} + +func (c *ChartDownloader) parseChartURL(ref string, version string) (*url.URL, error) { + u, err := url.Parse(ref) + if err != nil { + return nil, errors.Errorf("invalid chart URL format: %s", ref) + } + + if registry.IsOCI(u.String()) { + tag, err := c.getOciTag(ref, version) + + if err != nil { + return nil, err + } + + u.Path = fmt.Sprintf("%s:%s", u.Path, tag) + } + + return u, nil +} + +func (c *ChartDownloader) getOciTag(ref, version string) (string, error) { + var tag string + + // Evaluate whether an explicit version has been provided. Otherwise, determine version to use + _, errSemVer := semver.NewVersion(version) + if errSemVer == nil { + tag = version + } else { + // Retrieve list of repository tags + tags, err := c.RegistryClient.Tags(strings.TrimPrefix(ref, fmt.Sprintf("%s://", registry.OCIScheme))) + if err != nil { + return "", err + } + if len(tags) == 0 { + return "", errors.Errorf("Unable to locate any tags in provided repository: %s", ref) + } + + // Determine if version provided + // If empty, try to get the highest available tag + // If exact version, try to find it + // If semver constraint string, try to find a match + tag, err = registry.GetTagMatchingVersionOrConstraint(tags, version) + if err != nil { + return "", err + } + } + + return tag, nil +} + func loadRepoConfig(file string) (*repo.File, error) { r, err := repo.LoadFile(file) if err != nil && !errors.Is(err, fs.ErrNotExist) { diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index 3bb776473..fd3ce694f 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -328,16 +328,14 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { continue } - if m.SkipDownloadIfExists { - name := filepath.Base(churl) - if _, err = os.Stat(filepath.Join(destPath, name)); err == nil { - fmt.Fprintf(m.Out, "Already exists locally %s from repo %s\n", dep.Name, dep.Repository) - continue + version := "" + if registry.IsOCI(churl) { + churl, version, err = parseOCIRef(churl) + if err != nil { + return errors.Wrapf(err, "could not parse OCI reference") } } - fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) - dl := ChartDownloader{ Out: m.Out, Verify: m.Verify, @@ -355,7 +353,6 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { }, } - version := "" if registry.IsOCI(churl) { churl, version, err = parseOCIRef(churl) if err != nil { @@ -366,6 +363,21 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { getter.WithTagName(version)) } + if m.SkipDownloadIfExists { + u, err := dl.parseChartURL(churl, version) + + if err != nil { + return err + } + + name := dl.getChartName(u.String()) + if _, err = os.Stat(filepath.Join(destPath, name)); err == nil { + fmt.Fprintf(m.Out, "Already exists locally %s from repo %s\n", dep.Name, dep.Repository) + continue + } + } + + fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) if _, _, err = dl.DownloadTo(churl, version, tmpPath); err != nil { saveError = fmt.Errorf("could not download %s: %w", churl, err) break From 00d6c4ac508d4a726f370f51311830af6027ba81 Mon Sep 17 00:00:00 2001 From: Suleiman Dibirov Date: Wed, 4 Sep 2024 18:14:34 +0300 Subject: [PATCH 3/5] fixed manager.go Signed-off-by: Suleiman Dibirov --- pkg/downloader/manager.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index fd3ce694f..27474f583 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -27,6 +27,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "strings" "sync" @@ -275,6 +276,7 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { fmt.Fprintf(m.Out, "Saving %d charts\n", len(deps)) var saveError error churls := make(map[string]struct{}) + skippedCharts := make([]string, 0) for _, dep := range deps { // No repository means the chart is in charts directory if dep.Repository == "" { @@ -373,6 +375,8 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { name := dl.getChartName(u.String()) if _, err = os.Stat(filepath.Join(destPath, name)); err == nil { fmt.Fprintf(m.Out, "Already exists locally %s from repo %s\n", dep.Name, dep.Repository) + + skippedCharts = append(skippedCharts, name) continue } } @@ -389,7 +393,7 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { // TODO: this should probably be refactored to be a []error, so we can capture and provide more information rather than "last error wins". if saveError == nil { // now we can move all downloaded charts to destPath and delete outdated dependencies - if err := m.safeMoveDeps(deps, tmpPath, destPath); err != nil { + if err := m.safeMoveDeps(deps, tmpPath, destPath, skippedCharts); err != nil { return err } } else { @@ -416,13 +420,14 @@ func parseOCIRef(chartRef string) (string, string, error) { // It does this by first matching the file name to an expected pattern, then loading // the file to verify that it is a chart. // -// Any charts in dest that do not exist in source are removed (barring local dependencies) +// Any charts in dest that do not exist in source and not in skippedCharts are removed (barring local dependencies) +// skippedCharts is a list of charts that were skipped during the download process. // // Because it requires tar file introspection, it is more intensive than a basic move. // // This will only return errors that should stop processing entirely. Other errors // will emit log messages or be ignored. -func (m *Manager) safeMoveDeps(deps []*chart.Dependency, source, dest string) error { +func (m *Manager) safeMoveDeps(deps []*chart.Dependency, source, dest string, skippedCharts []string) error { existsInSourceDirectory := map[string]bool{} isLocalDependency := map[string]bool{} sourceFiles, err := os.ReadDir(source) @@ -463,7 +468,7 @@ func (m *Manager) safeMoveDeps(deps []*chart.Dependency, source, dest string) er fmt.Fprintln(m.Out, "Deleting outdated charts") // find all files that exist in dest that do not exist in source; delete them (outdated dependencies) for _, file := range destFiles { - if !file.IsDir() && !existsInSourceDirectory[file.Name()] { + if !file.IsDir() && !existsInSourceDirectory[file.Name()] && !slices.Contains(skippedCharts, file.Name()) { fname := filepath.Join(dest, file.Name()) ch, err := loader.LoadFile(fname) if err != nil { From 3f1e5a27a1da0ae9718dd3ab8966eb27099a249e Mon Sep 17 00:00:00 2001 From: Suleiman Dibirov Date: Thu, 11 Sep 2025 08:25:53 +0300 Subject: [PATCH 4/5] fixes after rebase Signed-off-by: Suleiman Dibirov --- pkg/downloader/chart_downloader.go | 11 ++++------- pkg/downloader/manager.go | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pkg/downloader/chart_downloader.go b/pkg/downloader/chart_downloader.go index fc70c5914..2b10dd4ee 100644 --- a/pkg/downloader/chart_downloader.go +++ b/pkg/downloader/chart_downloader.go @@ -29,6 +29,7 @@ import ( "path/filepath" "strings" + "github.com/Masterminds/semver/v3" "helm.sh/helm/v4/internal/fileutil" ifs "helm.sh/helm/v4/internal/third_party/dep/fs" "helm.sh/helm/v4/internal/urlutil" @@ -297,11 +298,7 @@ func (c *ChartDownloader) DownloadToCache(ref, version string) (string, *provena // Note, this does make an assumption that the name/version is unique to a // hash when a provenance file is used. If this isn't true, this section of code // will need to be reworked. - name := filepath.Base(u.Path) - if u.Scheme == registry.OCIScheme { - idx := strings.LastIndexByte(name, ':') - name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) - } + name := c.getChartName(u.String()) // Copy chart to a known location with the right name for verification and then // clean it up. @@ -583,7 +580,7 @@ func (c *ChartDownloader) getChartName(url string) string { func (c *ChartDownloader) parseChartURL(ref string, version string) (*url.URL, error) { u, err := url.Parse(ref) if err != nil { - return nil, errors.Errorf("invalid chart URL format: %s", ref) + return nil, fmt.Errorf("invalid chart URL format: %s", ref) } if registry.IsOCI(u.String()) { @@ -613,7 +610,7 @@ func (c *ChartDownloader) getOciTag(ref, version string) (string, error) { return "", err } if len(tags) == 0 { - return "", errors.Errorf("Unable to locate any tags in provided repository: %s", ref) + return "", fmt.Errorf("Unable to locate any tags in provided repository: %s", ref) } // Determine if version provided diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index 27474f583..8b7680e17 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -334,7 +334,7 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { if registry.IsOCI(churl) { churl, version, err = parseOCIRef(churl) if err != nil { - return errors.Wrapf(err, "could not parse OCI reference") + return fmt.Errorf("could not parse OCI reference: %w", err) } } From 4c9230adfeadaf9f3cdf0ea356f3a755191ab1a7 Mon Sep 17 00:00:00 2001 From: Suleiman Dibirov Date: Fri, 12 Sep 2025 14:23:48 +0300 Subject: [PATCH 5/5] lint fixes Signed-off-by: Suleiman Dibirov --- pkg/cmd/dependency_build.go | 20 ++++++++++---------- pkg/cmd/dependency_update.go | 20 ++++++++++---------- pkg/downloader/chart_downloader.go | 3 +-- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/pkg/cmd/dependency_build.go b/pkg/cmd/dependency_build.go index 08075845e..5946e76b6 100644 --- a/pkg/cmd/dependency_build.go +++ b/pkg/cmd/dependency_build.go @@ -61,17 +61,17 @@ func newDependencyBuildCmd(out io.Writer) *cobra.Command { } man := &downloader.Manager{ - Out: out, - ChartPath: chartpath, - Keyring: client.Keyring, - SkipUpdate: client.SkipRefresh, + Out: out, + ChartPath: chartpath, + Keyring: client.Keyring, + SkipUpdate: client.SkipRefresh, SkipDownloadIfExists: client.SkipDownloadIfExists, - Getters: getter.All(settings), - RegistryClient: registryClient, - RepositoryConfig: settings.RepositoryConfig, - RepositoryCache: settings.RepositoryCache, - ContentCache: settings.ContentCache, - Debug: settings.Debug, + Getters: getter.All(settings), + RegistryClient: registryClient, + RepositoryConfig: settings.RepositoryConfig, + RepositoryCache: settings.RepositoryCache, + ContentCache: settings.ContentCache, + Debug: settings.Debug, } if client.Verify { man.Verify = downloader.VerifyIfPossible diff --git a/pkg/cmd/dependency_update.go b/pkg/cmd/dependency_update.go index 714893ea1..346d8253d 100644 --- a/pkg/cmd/dependency_update.go +++ b/pkg/cmd/dependency_update.go @@ -65,17 +65,17 @@ func newDependencyUpdateCmd(_ *action.Configuration, out io.Writer) *cobra.Comma } man := &downloader.Manager{ - Out: out, - ChartPath: chartpath, - Keyring: client.Keyring, - SkipUpdate: client.SkipRefresh, + Out: out, + ChartPath: chartpath, + Keyring: client.Keyring, + SkipUpdate: client.SkipRefresh, SkipDownloadIfExists: client.SkipDownloadIfExists, - Getters: getter.All(settings), - RegistryClient: registryClient, - RepositoryConfig: settings.RepositoryConfig, - RepositoryCache: settings.RepositoryCache, - ContentCache: settings.ContentCache, - Debug: settings.Debug, + Getters: getter.All(settings), + RegistryClient: registryClient, + RepositoryConfig: settings.RepositoryConfig, + RepositoryCache: settings.RepositoryCache, + ContentCache: settings.ContentCache, + Debug: settings.Debug, } if client.Verify { man.Verify = downloader.VerifyAlways diff --git a/pkg/downloader/chart_downloader.go b/pkg/downloader/chart_downloader.go index 2b10dd4ee..be57f59bb 100644 --- a/pkg/downloader/chart_downloader.go +++ b/pkg/downloader/chart_downloader.go @@ -350,7 +350,6 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (string, *url return "", u, err } - if registry.IsOCI(u.String()) { if c.RegistryClient == nil { return "", nil, fmt.Errorf("unable to lookup ref %s at version '%s', missing registry client", ref, version) @@ -610,7 +609,7 @@ func (c *ChartDownloader) getOciTag(ref, version string) (string, error) { return "", err } if len(tags) == 0 { - return "", fmt.Errorf("Unable to locate any tags in provided repository: %s", ref) + return "", fmt.Errorf("unable to locate any tags in provided repository: %s", ref) } // Determine if version provided