diff --git a/pkg/action/dependency.go b/pkg/action/dependency.go index 03c370c8e..82b69b676 100644 --- a/pkg/action/dependency.go +++ b/pkg/action/dependency.go @@ -37,6 +37,7 @@ type Dependency struct { Verify bool Keyring string SkipRefresh bool + SkipDownloadIfExists bool ColumnWidth uint Username string Password string diff --git a/pkg/cmd/dependency.go b/pkg/cmd/dependency.go index 34bbff6be..587b16dc7 100644 --- a/pkg/cmd/dependency.go +++ b/pkg/cmd/dependency.go @@ -133,4 +133,5 @@ func addDependencySubcommandFlags(f *pflag.FlagSet, client *action.Dependency) { f.BoolVar(&client.InsecureSkipTLSverify, "insecure-skip-tls-verify", false, "skip tls certificate checks for the chart download") f.BoolVar(&client.PlainHTTP, "plain-http", false, "use insecure HTTP connections for the chart download") f.StringVar(&client.CaFile, "ca-file", "", "verify certificates of HTTPS-enabled servers using this CA bundle") + f.BoolVar(&client.SkipDownloadIfExists, "skip-download-if-exists", false, "skip download of the chart if it already exists in the charts directory") } diff --git a/pkg/cmd/dependency_build.go b/pkg/cmd/dependency_build.go index 320fe12ae..5946e76b6 100644 --- a/pkg/cmd/dependency_build.go +++ b/pkg/cmd/dependency_build.go @@ -61,16 +61,17 @@ func newDependencyBuildCmd(out io.Writer) *cobra.Command { } man := &downloader.Manager{ - Out: out, - ChartPath: chartpath, - Keyring: client.Keyring, - SkipUpdate: client.SkipRefresh, - Getters: getter.All(settings), - RegistryClient: registryClient, - RepositoryConfig: settings.RepositoryConfig, - RepositoryCache: settings.RepositoryCache, - ContentCache: settings.ContentCache, - Debug: settings.Debug, + Out: out, + ChartPath: chartpath, + Keyring: client.Keyring, + SkipUpdate: client.SkipRefresh, + SkipDownloadIfExists: client.SkipDownloadIfExists, + Getters: getter.All(settings), + RegistryClient: registryClient, + RepositoryConfig: settings.RepositoryConfig, + RepositoryCache: settings.RepositoryCache, + ContentCache: settings.ContentCache, + Debug: settings.Debug, } if client.Verify { man.Verify = downloader.VerifyIfPossible diff --git a/pkg/cmd/dependency_update.go b/pkg/cmd/dependency_update.go index b534fb48a..346d8253d 100644 --- a/pkg/cmd/dependency_update.go +++ b/pkg/cmd/dependency_update.go @@ -65,16 +65,17 @@ func newDependencyUpdateCmd(_ *action.Configuration, out io.Writer) *cobra.Comma } man := &downloader.Manager{ - Out: out, - ChartPath: chartpath, - Keyring: client.Keyring, - SkipUpdate: client.SkipRefresh, - Getters: getter.All(settings), - RegistryClient: registryClient, - RepositoryConfig: settings.RepositoryConfig, - RepositoryCache: settings.RepositoryCache, - ContentCache: settings.ContentCache, - Debug: settings.Debug, + Out: out, + ChartPath: chartpath, + Keyring: client.Keyring, + SkipUpdate: client.SkipRefresh, + SkipDownloadIfExists: client.SkipDownloadIfExists, + Getters: getter.All(settings), + RegistryClient: registryClient, + RepositoryConfig: settings.RepositoryConfig, + RepositoryCache: settings.RepositoryCache, + ContentCache: settings.ContentCache, + Debug: settings.Debug, } if client.Verify { man.Verify = downloader.VerifyAlways diff --git a/pkg/downloader/chart_downloader.go b/pkg/downloader/chart_downloader.go index 00c8c56e8..be57f59bb 100644 --- a/pkg/downloader/chart_downloader.go +++ b/pkg/downloader/chart_downloader.go @@ -29,6 +29,7 @@ import ( "path/filepath" "strings" + "github.com/Masterminds/semver/v3" "helm.sh/helm/v4/internal/fileutil" ifs "helm.sh/helm/v4/internal/third_party/dep/fs" "helm.sh/helm/v4/internal/urlutil" @@ -149,12 +150,7 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven } } - name := filepath.Base(u.Path) - if u.Scheme == registry.OCIScheme { - idx := strings.LastIndexByte(name, ':') - name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) - } - + name := c.getChartName(u.String()) destfile := filepath.Join(dest, name) if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil { return destfile, nil, err @@ -302,11 +298,7 @@ func (c *ChartDownloader) DownloadToCache(ref, version string) (string, *provena // Note, this does make an assumption that the name/version is unique to a // hash when a provenance file is used. If this isn't true, this section of code // will need to be reworked. - name := filepath.Base(u.Path) - if u.Scheme == registry.OCIScheme { - idx := strings.LastIndexByte(name, ':') - name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) - } + name := c.getChartName(u.String()) // Copy chart to a known location with the right name for verification and then // clean it up. @@ -353,9 +345,9 @@ func (c *ChartDownloader) DownloadToCache(ref, version string) (string, *provena // // TODO: support OCI hash func (c *ChartDownloader) ResolveChartVersion(ref, version string) (string, *url.URL, error) { - u, err := url.Parse(ref) + u, err := c.parseChartURL(ref, version) if err != nil { - return "", nil, fmt.Errorf("invalid chart URL format: %s", ref) + return "", u, err } if registry.IsOCI(u.String()) { @@ -574,6 +566,65 @@ func (c *ChartDownloader) scanReposForURL(u string, rf *repo.File) (*repo.Entry, return nil, ErrNoOwnerRepo } +func (c *ChartDownloader) getChartName(url string) string { + name := filepath.Base(url) + if registry.IsOCI(url) { + idx := strings.LastIndexByte(name, ':') + name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) + } + + return name +} + +func (c *ChartDownloader) parseChartURL(ref string, version string) (*url.URL, error) { + u, err := url.Parse(ref) + if err != nil { + return nil, fmt.Errorf("invalid chart URL format: %s", ref) + } + + if registry.IsOCI(u.String()) { + tag, err := c.getOciTag(ref, version) + + if err != nil { + return nil, err + } + + u.Path = fmt.Sprintf("%s:%s", u.Path, tag) + } + + return u, nil +} + +func (c *ChartDownloader) getOciTag(ref, version string) (string, error) { + var tag string + + // Evaluate whether an explicit version has been provided. Otherwise, determine version to use + _, errSemVer := semver.NewVersion(version) + if errSemVer == nil { + tag = version + } else { + // Retrieve list of repository tags + tags, err := c.RegistryClient.Tags(strings.TrimPrefix(ref, fmt.Sprintf("%s://", registry.OCIScheme))) + if err != nil { + return "", err + } + if len(tags) == 0 { + return "", fmt.Errorf("unable to locate any tags in provided repository: %s", ref) + } + + // Determine if version provided + // If empty, try to get the highest available tag + // If exact version, try to find it + // If semver constraint string, try to find a match + tag, err = registry.GetTagMatchingVersionOrConstraint(tags, version) + if err != nil { + return "", err + } + } + + return tag, nil +} + func loadRepoConfig(file string) (*repo.File, error) { r, err := repo.LoadFile(file) if err != nil && !errors.Is(err, fs.ErrNotExist) { diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index d41b8fdb4..8b7680e17 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -27,6 +27,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "strings" "sync" @@ -70,6 +71,8 @@ type Manager struct { Keyring string // SkipUpdate indicates that the repository should not be updated first. SkipUpdate bool + // SkipDownloadIfExists indicates that the chart should not be downloaded if it already exists. + SkipDownloadIfExists bool // Getter collection for the operation Getters []getter.Provider RegistryClient *registry.Client @@ -273,6 +276,7 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { fmt.Fprintf(m.Out, "Saving %d charts\n", len(deps)) var saveError error churls := make(map[string]struct{}) + skippedCharts := make([]string, 0) for _, dep := range deps { // No repository means the chart is in charts directory if dep.Repository == "" { @@ -326,7 +330,13 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { continue } - fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) + version := "" + if registry.IsOCI(churl) { + churl, version, err = parseOCIRef(churl) + if err != nil { + return fmt.Errorf("could not parse OCI reference: %w", err) + } + } dl := ChartDownloader{ Out: m.Out, @@ -345,7 +355,6 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { }, } - version := "" if registry.IsOCI(churl) { churl, version, err = parseOCIRef(churl) if err != nil { @@ -356,6 +365,23 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { getter.WithTagName(version)) } + if m.SkipDownloadIfExists { + u, err := dl.parseChartURL(churl, version) + + if err != nil { + return err + } + + name := dl.getChartName(u.String()) + if _, err = os.Stat(filepath.Join(destPath, name)); err == nil { + fmt.Fprintf(m.Out, "Already exists locally %s from repo %s\n", dep.Name, dep.Repository) + + skippedCharts = append(skippedCharts, name) + continue + } + } + + fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) if _, _, err = dl.DownloadTo(churl, version, tmpPath); err != nil { saveError = fmt.Errorf("could not download %s: %w", churl, err) break @@ -367,7 +393,7 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { // TODO: this should probably be refactored to be a []error, so we can capture and provide more information rather than "last error wins". if saveError == nil { // now we can move all downloaded charts to destPath and delete outdated dependencies - if err := m.safeMoveDeps(deps, tmpPath, destPath); err != nil { + if err := m.safeMoveDeps(deps, tmpPath, destPath, skippedCharts); err != nil { return err } } else { @@ -394,13 +420,14 @@ func parseOCIRef(chartRef string) (string, string, error) { // It does this by first matching the file name to an expected pattern, then loading // the file to verify that it is a chart. // -// Any charts in dest that do not exist in source are removed (barring local dependencies) +// Any charts in dest that do not exist in source and not in skippedCharts are removed (barring local dependencies) +// skippedCharts is a list of charts that were skipped during the download process. // // Because it requires tar file introspection, it is more intensive than a basic move. // // This will only return errors that should stop processing entirely. Other errors // will emit log messages or be ignored. -func (m *Manager) safeMoveDeps(deps []*chart.Dependency, source, dest string) error { +func (m *Manager) safeMoveDeps(deps []*chart.Dependency, source, dest string, skippedCharts []string) error { existsInSourceDirectory := map[string]bool{} isLocalDependency := map[string]bool{} sourceFiles, err := os.ReadDir(source) @@ -441,7 +468,7 @@ func (m *Manager) safeMoveDeps(deps []*chart.Dependency, source, dest string) er fmt.Fprintln(m.Out, "Deleting outdated charts") // find all files that exist in dest that do not exist in source; delete them (outdated dependencies) for _, file := range destFiles { - if !file.IsDir() && !existsInSourceDirectory[file.Name()] { + if !file.IsDir() && !existsInSourceDirectory[file.Name()] && !slices.Contains(skippedCharts, file.Name()) { fname := filepath.Join(dest, file.Name()) ch, err := loader.LoadFile(fname) if err != nil {