Merge c7a955c4af into d1e9c022c6

pkg/downloader/chart_downloader.go

@@ -143,6 +143,73 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven
 	return destfile, ver, nil
 }
 
+func (c *ChartDownloader) DownloadAllTo(downloadEntries []DownloadEntry, dest string) ([]string, []*provenance.Verification, error) {
+	indexFileCache := make(map[string]*repo.IndexFile)
+	var destFiles []string
+	var verifications []*provenance.Verification
+	for _, downloadEntry := range downloadEntries {
+		u, err := c.resolveChartVersionWithIndexFileCache(downloadEntry.Ref, downloadEntry.Version, indexFileCache)
+		if err != nil {
+			return destFiles, verifications, err
+		}
+
+		g, err := c.Getters.ByScheme(u.Scheme)
+		if err != nil {
+			return destFiles, verifications, err
+		}
+
+		data, err := g.Get(u.String(), c.Options...)
+		if err != nil {
+			return destFiles, verifications, err
+		}
+
+		name := filepath.Base(u.Path)
+		if u.Scheme == registry.OCIScheme {
+			idx := strings.LastIndexByte(name, ':')
+			name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:])
+		}
+
+		destfile := filepath.Join(dest, name)
+		destFiles = append(destFiles, destfile)
+		if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil {
+			return destFiles, verifications, err
+		}
+
+		// If provenance is requested, verify it.
+		ver := &provenance.Verification{}
+		if c.Verify > VerifyNever {
+			body, err := g.Get(u.String() + ".prov")
+			if err != nil {
+				if c.Verify == VerifyAlways {
+					return destFiles, verifications, errors.Errorf("failed to fetch provenance %q", u.String()+".prov")
+				}
+				fmt.Fprintf(c.Out, "WARNING: Verification not found for %s: %s\n", downloadEntry.Ref, err)
+				continue
+			}
+			provfile := destfile + ".prov"
+			if err := fileutil.AtomicWriteFile(provfile, body, 0644); err != nil {
+				return destFiles, verifications, err
+			}
+
+			if c.Verify != VerifyLater {
+				ver, err = VerifyChart(destfile, c.Keyring)
+				if err != nil {
+					// Fail always in this case, since it means the verification step
+					// failed.
+					return destFiles, verifications, err
+				}
+			}
+		}
+		verifications = append(verifications, ver)
+	}
+	return destFiles, verifications, nil
+}
+
+type DownloadEntry struct {
+	Ref     string
+	Version string
+}
+
 func (c *ChartDownloader) getOciURI(ref, version string, u *url.URL) (*url.URL, error) {
 	var tag string
 	var err error
@@ -192,6 +259,10 @@ func (c *ChartDownloader) getOciURI(ref, version string, u *url.URL) (*url.URL,
 //   - If version is empty, this will return the URL for the latest version
 //   - If no version can be found, an error is returned
 func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, error) {
+	return c.resolveChartVersionWithIndexFileCache(ref, version, make(map[string]*repo.IndexFile))
+}
+
+func (c *ChartDownloader) resolveChartVersionWithIndexFileCache(ref, version string, indexFileCache map[string]*repo.IndexFile) (*url.URL, error) {
 	u, err := url.Parse(ref)
 	if err != nil {
 		return nil, errors.Errorf("invalid chart URL format: %s", ref)
@@ -213,7 +284,7 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, er
 		// we want to find the repo in case we have special SSL cert config
 		// for that repo.
 
-		rc, err := c.scanReposForURL(ref, rf)
+		rc, err := c.scanReposForURLWithIndexFileCache(ref, rf, indexFileCache)
 		if err != nil {
 			// If there is no special config, return the default HTTP client and
 			// swallow the error.
@@ -371,6 +442,12 @@ func pickChartRepositoryConfigByName(name string, cfgs []*repo.Entry) (*repo.Ent
 // will return the first one it finds. Order is determined by the order of repositories
 // in the repositories.yaml file.
 func (c *ChartDownloader) scanReposForURL(u string, rf *repo.File) (*repo.Entry, error) {
+	// FIXME: This is far from optimal. Larger installations and index files will
+	// incur a performance hit for this type of scanning.
+	return c.scanReposForURLWithIndexFileCache(u, rf, make(map[string]*repo.IndexFile))
+}
+
+func (c *ChartDownloader) scanReposForURLWithIndexFileCache(u string, rf *repo.File, indexFileCache map[string]*repo.IndexFile) (*repo.Entry, error) {
 	// FIXME: This is far from optimal. Larger installations and index files will
 	// incur a performance hit for this type of scanning.
 	for _, rc := range rf.Repositories {
@@ -380,10 +457,15 @@ func (c *ChartDownloader) scanReposForURL(u string, rf *repo.File) (*repo.Entry,
 		}
 
 		idxFile := filepath.Join(c.RepositoryCache, helmpath.CacheIndexFile(r.Config.Name))
-		i, err := repo.LoadIndexFile(idxFile)
+		i, ok := indexFileCache[idxFile]
+		if !ok {
+			var err error
+			i, err = repo.LoadIndexFile(idxFile)
 			if err != nil {
 				return nil, errors.Wrap(err, "no cached repo found. (try 'helm repo update')")
 			}
+			indexFileCache[idxFile] = i
+		}
 
 		for _, entry := range i.Entries {
 			for _, ver := range entry {

pkg/downloader/manager.go

@@ -197,7 +197,7 @@ func (m *Manager) Update() error {
 	}
 
 	// Now we need to fetch every package here into charts/
-	if err := m.downloadAll(lock.Dependencies); err != nil {
+	if err := m.batchDownloadAll(lock.Dependencies); err != nil {
 		return err
 	}
 
@@ -373,6 +373,181 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error {
 	return nil
 }
 
+func (m *Manager) batchDownloadAll(deps []*chart.Dependency) error {
+	repos, err := m.loadChartRepositories()
+	if err != nil {
+		return err
+	}
+
+	destPath := filepath.Join(m.ChartPath, "charts")
+	tmpPath := filepath.Join(m.ChartPath, "tmpcharts")
+
+	// Check if 'charts' directory is not actally a directory. If it does not exist, create it.
+	if fi, err := os.Stat(destPath); err == nil {
+		if !fi.IsDir() {
+			return errors.Errorf("%q is not a directory", destPath)
+		}
+	} else if os.IsNotExist(err) {
+		if err := os.MkdirAll(destPath, 0755); err != nil {
+			return err
+		}
+	} else {
+		return fmt.Errorf("unable to retrieve file info for '%s': %v", destPath, err)
+	}
+
+	// Prepare tmpPath
+	if err := os.MkdirAll(tmpPath, 0755); err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpPath)
+
+	fmt.Fprintf(m.Out, "Saving %d charts\n", len(deps))
+	var saveError error
+	churls := make(map[string]struct{})
+	chartDownloaderCache := make(map[ChartDownloaderCacheKey][]DownloadEntry)
+	for _, dep := range deps {
+		// No repository means the chart is in charts directory
+		if dep.Repository == "" {
+			fmt.Fprintf(m.Out, "Dependency %s did not declare a repository. Assuming it exists in the charts directory\n", dep.Name)
+			// NOTE: we are only validating the local dependency conforms to the constraints. No copying to tmpPath is necessary.
+			chartPath := filepath.Join(destPath, dep.Name)
+			ch, err := loader.LoadDir(chartPath)
+			if err != nil {
+				return fmt.Errorf("unable to load chart '%s': %v", chartPath, err)
+			}
+
+			constraint, err := semver.NewConstraint(dep.Version)
+			if err != nil {
+				return fmt.Errorf("dependency %s has an invalid version/constraint format: %s", dep.Name, err)
+			}
+
+			v, err := semver.NewVersion(ch.Metadata.Version)
+			if err != nil {
+				return fmt.Errorf("invalid version %s for dependency %s: %s", dep.Version, dep.Name, err)
+			}
+
+			if !constraint.Check(v) {
+				saveError = fmt.Errorf("dependency %s at version %s does not satisfy the constraint %s", dep.Name, ch.Metadata.Version, dep.Version)
+				break
+			}
+			continue
+		}
+		if strings.HasPrefix(dep.Repository, "file://") {
+			if m.Debug {
+				fmt.Fprintf(m.Out, "Archiving %s from repo %s\n", dep.Name, dep.Repository)
+			}
+			ver, err := tarFromLocalDir(m.ChartPath, dep.Name, dep.Repository, dep.Version, tmpPath)
+			if err != nil {
+				saveError = err
+				break
+			}
+			dep.Version = ver
+			continue
+		}
+
+		// Any failure to resolve/download a chart should fail:
+		// https://github.com/helm/helm/issues/1439
+		churl, username, password, insecureskiptlsverify, passcredentialsall, caFile, certFile, keyFile, err := m.findChartURL(dep.Name, dep.Version, dep.Repository, repos)
+		if err != nil {
+			saveError = errors.Wrapf(err, "could not find %s", churl)
+			break
+		}
+
+		if _, ok := churls[churl]; ok {
+			fmt.Fprintf(m.Out, "Already processed %s from repo %s\n", dep.Name, dep.Repository)
+			continue
+		}
+
+		version := ""
+		isOCI := false
+		if registry.IsOCI(churl) {
+			churl, version, err = parseOCIRef(churl)
+			if err != nil {
+				return errors.Wrapf(err, "could not parse OCI reference")
+			}
+			isOCI = true
+		}
+
+		chartDownloaderCacheKey := ChartDownloaderCacheKey{
+			Username:              username,
+			Password:              password,
+			InSecureSkipTLSVerify: insecureskiptlsverify,
+			PassCredentialsAll:    passcredentialsall,
+			CAFile:                caFile,
+			CertFile:              certFile,
+			KeyFile:               keyFile,
+			IsOCI:                 isOCI,
+			Version:               version,
+		}
+		if _, ok := chartDownloaderCache[chartDownloaderCacheKey]; !ok {
+			chartDownloaderCache[chartDownloaderCacheKey] = []DownloadEntry{}
+		}
+		chartDownloaderCache[chartDownloaderCacheKey] = append(chartDownloaderCache[chartDownloaderCacheKey], DownloadEntry{
+			Ref:     churl,
+			Version: version,
+		})
+		churls[churl] = struct{}{}
+	}
+
+	if saveError == nil {
+		for chartDownloaderCacheKey, downloadEntries := range chartDownloaderCache {
+
+			//fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository)
+
+			dl := ChartDownloader{
+				Out:              m.Out,
+				Verify:           m.Verify,
+				Keyring:          m.Keyring,
+				RepositoryConfig: m.RepositoryConfig,
+				RepositoryCache:  m.RepositoryCache,
+				RegistryClient:   m.RegistryClient,
+				Getters:          m.Getters,
+				Options: []getter.Option{
+					getter.WithBasicAuth(chartDownloaderCacheKey.Username, chartDownloaderCacheKey.Password),
+					getter.WithPassCredentialsAll(chartDownloaderCacheKey.PassCredentialsAll),
+					getter.WithInsecureSkipVerifyTLS(chartDownloaderCacheKey.InSecureSkipTLSVerify),
+					getter.WithTLSClientConfig(chartDownloaderCacheKey.CertFile, chartDownloaderCacheKey.KeyFile, chartDownloaderCacheKey.CAFile),
+				},
+			}
+
+			if chartDownloaderCacheKey.IsOCI {
+				dl.Options = append(dl.Options,
+					getter.WithRegistryClient(m.RegistryClient),
+					getter.WithTagName(chartDownloaderCacheKey.Version))
+			}
+
+			if _, _, err = dl.DownloadAllTo(downloadEntries, tmpPath); err != nil {
+				saveError = errors.Wrapf(err, "could not download %v", downloadEntries)
+				break
+			}
+		}
+	}
+
+	// TODO: this should probably be refactored to be a []error, so we can capture and provide more information rather than "last error wins".
+	if saveError == nil {
+		// now we can move all downloaded charts to destPath and delete outdated dependencies
+		if err := m.safeMoveDeps(deps, tmpPath, destPath); err != nil {
+			return err
+		}
+	} else {
+		fmt.Fprintln(m.Out, "Save error occurred: ", saveError)
+		return saveError
+	}
+	return nil
+}
+
+type ChartDownloaderCacheKey struct {
+	Username              string
+	Password              string
+	InSecureSkipTLSVerify bool
+	PassCredentialsAll    bool
+	CAFile                string
+	CertFile              string
+	KeyFile               string
+	IsOCI                 bool
+	Version               string
+}
+
 func parseOCIRef(chartRef string) (string, string, error) {
 	refTagRegexp := regexp.MustCompile(`^(oci://[^:]+(:[0-9]{1,5})?[^:]+):(.*)$`)
 	caps := refTagRegexp.FindStringSubmatch(chartRef)

pkg/downloader/manager_test.go

@@ -290,6 +290,54 @@ version: 0.1.0`
 	}
 }
 
+func TestBatchDownloadAll(t *testing.T) {
+	chartPath := t.TempDir()
+	m := &Manager{
+		Out:              new(bytes.Buffer),
+		RepositoryConfig: repoConfig,
+		RepositoryCache:  repoCache,
+		ChartPath:        chartPath,
+	}
+	signtest, err := loader.LoadDir(filepath.Join("testdata", "signtest"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := chartutil.SaveDir(signtest, filepath.Join(chartPath, "testdata")); err != nil {
+		t.Fatal(err)
+	}
+
+	local, err := loader.LoadDir(filepath.Join("testdata", "local-subchart"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := chartutil.SaveDir(local, filepath.Join(chartPath, "charts")); err != nil {
+		t.Fatal(err)
+	}
+
+	signDep := &chart.Dependency{
+		Name:       signtest.Name(),
+		Repository: "file://./testdata/signtest",
+		Version:    signtest.Metadata.Version,
+	}
+	localDep := &chart.Dependency{
+		Name:       local.Name(),
+		Repository: "",
+		Version:    local.Metadata.Version,
+	}
+
+	// create a 'tmpcharts' directory to test #5567
+	if err := os.MkdirAll(filepath.Join(chartPath, "tmpcharts"), 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := m.batchDownloadAll([]*chart.Dependency{signDep, localDep}); err != nil {
+		t.Error(err)
+	}
+
+	if _, err := os.Stat(filepath.Join(chartPath, "charts", "signtest-0.1.0.tgz")); os.IsNotExist(err) {
+		t.Error(err)
+	}
+}
+
 func TestUpdateBeforeBuild(t *testing.T) {
 	// Set up a fake repo
 	srv, err := repotest.NewTempServerWithCleanup(t, "testdata/*.tgz*")