diff --git a/pkg/downloader/chart_downloader.go b/pkg/downloader/chart_downloader.go index dde6a1057..9eded52c2 100644 --- a/pkg/downloader/chart_downloader.go +++ b/pkg/downloader/chart_downloader.go @@ -143,6 +143,73 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven return destfile, ver, nil } +func (c *ChartDownloader) DownloadAllTo(downloadEntries []DownloadEntry, dest string) ([]string, []*provenance.Verification, error) { + indexFileCache := make(map[string]*repo.IndexFile) + var destFiles []string + var verifications []*provenance.Verification + for _, downloadEntry := range downloadEntries { + u, err := c.resolveChartVersionWithIndexFileCache(downloadEntry.Ref, downloadEntry.Version, indexFileCache) + if err != nil { + return destFiles, verifications, err + } + + g, err := c.Getters.ByScheme(u.Scheme) + if err != nil { + return destFiles, verifications, err + } + + data, err := g.Get(u.String(), c.Options...) + if err != nil { + return destFiles, verifications, err + } + + name := filepath.Base(u.Path) + if u.Scheme == registry.OCIScheme { + idx := strings.LastIndexByte(name, ':') + name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) + } + + destfile := filepath.Join(dest, name) + destFiles = append(destFiles, destfile) + if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil { + return destFiles, verifications, err + } + + // If provenance is requested, verify it. + ver := &provenance.Verification{} + if c.Verify > VerifyNever { + body, err := g.Get(u.String() + ".prov") + if err != nil { + if c.Verify == VerifyAlways { + return destFiles, verifications, errors.Errorf("failed to fetch provenance %q", u.String()+".prov") + } + fmt.Fprintf(c.Out, "WARNING: Verification not found for %s: %s\n", downloadEntry.Ref, err) + continue + } + provfile := destfile + ".prov" + if err := fileutil.AtomicWriteFile(provfile, body, 0644); err != nil { + return destFiles, verifications, err + } + + if c.Verify != VerifyLater { + ver, err = VerifyChart(destfile, c.Keyring) + if err != nil { + // Fail always in this case, since it means the verification step + // failed. + return destFiles, verifications, err + } + } + } + verifications = append(verifications, ver) + } + return destFiles, verifications, nil +} + +type DownloadEntry struct { + Ref string + Version string +} + func (c *ChartDownloader) getOciURI(ref, version string, u *url.URL) (*url.URL, error) { var tag string var err error @@ -192,6 +259,10 @@ func (c *ChartDownloader) getOciURI(ref, version string, u *url.URL) (*url.URL, // - If version is empty, this will return the URL for the latest version // - If no version can be found, an error is returned func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, error) { + return c.resolveChartVersionWithIndexFileCache(ref, version, make(map[string]*repo.IndexFile)) +} + +func (c *ChartDownloader) resolveChartVersionWithIndexFileCache(ref, version string, indexFileCache map[string]*repo.IndexFile) (*url.URL, error) { u, err := url.Parse(ref) if err != nil { return nil, errors.Errorf("invalid chart URL format: %s", ref) @@ -213,7 +284,7 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, er // we want to find the repo in case we have special SSL cert config // for that repo. - rc, err := c.scanReposForURL(ref, rf) + rc, err := c.scanReposForURLWithIndexFileCache(ref, rf, indexFileCache) if err != nil { // If there is no special config, return the default HTTP client and // swallow the error. @@ -371,6 +442,12 @@ func pickChartRepositoryConfigByName(name string, cfgs []*repo.Entry) (*repo.Ent // will return the first one it finds. Order is determined by the order of repositories // in the repositories.yaml file. func (c *ChartDownloader) scanReposForURL(u string, rf *repo.File) (*repo.Entry, error) { + // FIXME: This is far from optimal. Larger installations and index files will + // incur a performance hit for this type of scanning. + return c.scanReposForURLWithIndexFileCache(u, rf, make(map[string]*repo.IndexFile)) +} + +func (c *ChartDownloader) scanReposForURLWithIndexFileCache(u string, rf *repo.File, indexFileCache map[string]*repo.IndexFile) (*repo.Entry, error) { // FIXME: This is far from optimal. Larger installations and index files will // incur a performance hit for this type of scanning. for _, rc := range rf.Repositories { @@ -380,9 +457,14 @@ func (c *ChartDownloader) scanReposForURL(u string, rf *repo.File) (*repo.Entry, } idxFile := filepath.Join(c.RepositoryCache, helmpath.CacheIndexFile(r.Config.Name)) - i, err := repo.LoadIndexFile(idxFile) - if err != nil { - return nil, errors.Wrap(err, "no cached repo found. (try 'helm repo update')") + i, ok := indexFileCache[idxFile] + if !ok { + var err error + i, err = repo.LoadIndexFile(idxFile) + if err != nil { + return nil, errors.Wrap(err, "no cached repo found. (try 'helm repo update')") + } + indexFileCache[idxFile] = i } for _, entry := range i.Entries { diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index ec4056d27..3982891eb 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -197,7 +197,7 @@ func (m *Manager) Update() error { } // Now we need to fetch every package here into charts/ - if err := m.downloadAll(lock.Dependencies); err != nil { + if err := m.batchDownloadAll(lock.Dependencies); err != nil { return err } @@ -373,6 +373,181 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { return nil } +func (m *Manager) batchDownloadAll(deps []*chart.Dependency) error { + repos, err := m.loadChartRepositories() + if err != nil { + return err + } + + destPath := filepath.Join(m.ChartPath, "charts") + tmpPath := filepath.Join(m.ChartPath, "tmpcharts") + + // Check if 'charts' directory is not actally a directory. If it does not exist, create it. + if fi, err := os.Stat(destPath); err == nil { + if !fi.IsDir() { + return errors.Errorf("%q is not a directory", destPath) + } + } else if os.IsNotExist(err) { + if err := os.MkdirAll(destPath, 0755); err != nil { + return err + } + } else { + return fmt.Errorf("unable to retrieve file info for '%s': %v", destPath, err) + } + + // Prepare tmpPath + if err := os.MkdirAll(tmpPath, 0755); err != nil { + return err + } + defer os.RemoveAll(tmpPath) + + fmt.Fprintf(m.Out, "Saving %d charts\n", len(deps)) + var saveError error + churls := make(map[string]struct{}) + chartDownloaderCache := make(map[ChartDownloaderCacheKey][]DownloadEntry) + for _, dep := range deps { + // No repository means the chart is in charts directory + if dep.Repository == "" { + fmt.Fprintf(m.Out, "Dependency %s did not declare a repository. Assuming it exists in the charts directory\n", dep.Name) + // NOTE: we are only validating the local dependency conforms to the constraints. No copying to tmpPath is necessary. + chartPath := filepath.Join(destPath, dep.Name) + ch, err := loader.LoadDir(chartPath) + if err != nil { + return fmt.Errorf("unable to load chart '%s': %v", chartPath, err) + } + + constraint, err := semver.NewConstraint(dep.Version) + if err != nil { + return fmt.Errorf("dependency %s has an invalid version/constraint format: %s", dep.Name, err) + } + + v, err := semver.NewVersion(ch.Metadata.Version) + if err != nil { + return fmt.Errorf("invalid version %s for dependency %s: %s", dep.Version, dep.Name, err) + } + + if !constraint.Check(v) { + saveError = fmt.Errorf("dependency %s at version %s does not satisfy the constraint %s", dep.Name, ch.Metadata.Version, dep.Version) + break + } + continue + } + if strings.HasPrefix(dep.Repository, "file://") { + if m.Debug { + fmt.Fprintf(m.Out, "Archiving %s from repo %s\n", dep.Name, dep.Repository) + } + ver, err := tarFromLocalDir(m.ChartPath, dep.Name, dep.Repository, dep.Version, tmpPath) + if err != nil { + saveError = err + break + } + dep.Version = ver + continue + } + + // Any failure to resolve/download a chart should fail: + // https://github.com/helm/helm/issues/1439 + churl, username, password, insecureskiptlsverify, passcredentialsall, caFile, certFile, keyFile, err := m.findChartURL(dep.Name, dep.Version, dep.Repository, repos) + if err != nil { + saveError = errors.Wrapf(err, "could not find %s", churl) + break + } + + if _, ok := churls[churl]; ok { + fmt.Fprintf(m.Out, "Already processed %s from repo %s\n", dep.Name, dep.Repository) + continue + } + + version := "" + isOCI := false + if registry.IsOCI(churl) { + churl, version, err = parseOCIRef(churl) + if err != nil { + return errors.Wrapf(err, "could not parse OCI reference") + } + isOCI = true + } + + chartDownloaderCacheKey := ChartDownloaderCacheKey{ + Username: username, + Password: password, + InSecureSkipTLSVerify: insecureskiptlsverify, + PassCredentialsAll: passcredentialsall, + CAFile: caFile, + CertFile: certFile, + KeyFile: keyFile, + IsOCI: isOCI, + Version: version, + } + if _, ok := chartDownloaderCache[chartDownloaderCacheKey]; !ok { + chartDownloaderCache[chartDownloaderCacheKey] = []DownloadEntry{} + } + chartDownloaderCache[chartDownloaderCacheKey] = append(chartDownloaderCache[chartDownloaderCacheKey], DownloadEntry{ + Ref: churl, + Version: version, + }) + churls[churl] = struct{}{} + } + + if saveError == nil { + for chartDownloaderCacheKey, downloadEntries := range chartDownloaderCache { + + //fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) + + dl := ChartDownloader{ + Out: m.Out, + Verify: m.Verify, + Keyring: m.Keyring, + RepositoryConfig: m.RepositoryConfig, + RepositoryCache: m.RepositoryCache, + RegistryClient: m.RegistryClient, + Getters: m.Getters, + Options: []getter.Option{ + getter.WithBasicAuth(chartDownloaderCacheKey.Username, chartDownloaderCacheKey.Password), + getter.WithPassCredentialsAll(chartDownloaderCacheKey.PassCredentialsAll), + getter.WithInsecureSkipVerifyTLS(chartDownloaderCacheKey.InSecureSkipTLSVerify), + getter.WithTLSClientConfig(chartDownloaderCacheKey.CertFile, chartDownloaderCacheKey.KeyFile, chartDownloaderCacheKey.CAFile), + }, + } + + if chartDownloaderCacheKey.IsOCI { + dl.Options = append(dl.Options, + getter.WithRegistryClient(m.RegistryClient), + getter.WithTagName(chartDownloaderCacheKey.Version)) + } + + if _, _, err = dl.DownloadAllTo(downloadEntries, tmpPath); err != nil { + saveError = errors.Wrapf(err, "could not download %v", downloadEntries) + break + } + } + } + + // TODO: this should probably be refactored to be a []error, so we can capture and provide more information rather than "last error wins". + if saveError == nil { + // now we can move all downloaded charts to destPath and delete outdated dependencies + if err := m.safeMoveDeps(deps, tmpPath, destPath); err != nil { + return err + } + } else { + fmt.Fprintln(m.Out, "Save error occurred: ", saveError) + return saveError + } + return nil +} + +type ChartDownloaderCacheKey struct { + Username string + Password string + InSecureSkipTLSVerify bool + PassCredentialsAll bool + CAFile string + CertFile string + KeyFile string + IsOCI bool + Version string +} + func parseOCIRef(chartRef string) (string, string, error) { refTagRegexp := regexp.MustCompile(`^(oci://[^:]+(:[0-9]{1,5})?[^:]+):(.*)$`) caps := refTagRegexp.FindStringSubmatch(chartRef) diff --git a/pkg/downloader/manager_test.go b/pkg/downloader/manager_test.go index db2487d16..84d15f869 100644 --- a/pkg/downloader/manager_test.go +++ b/pkg/downloader/manager_test.go @@ -290,6 +290,54 @@ version: 0.1.0` } } +func TestBatchDownloadAll(t *testing.T) { + chartPath := t.TempDir() + m := &Manager{ + Out: new(bytes.Buffer), + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + ChartPath: chartPath, + } + signtest, err := loader.LoadDir(filepath.Join("testdata", "signtest")) + if err != nil { + t.Fatal(err) + } + if err := chartutil.SaveDir(signtest, filepath.Join(chartPath, "testdata")); err != nil { + t.Fatal(err) + } + + local, err := loader.LoadDir(filepath.Join("testdata", "local-subchart")) + if err != nil { + t.Fatal(err) + } + if err := chartutil.SaveDir(local, filepath.Join(chartPath, "charts")); err != nil { + t.Fatal(err) + } + + signDep := &chart.Dependency{ + Name: signtest.Name(), + Repository: "file://./testdata/signtest", + Version: signtest.Metadata.Version, + } + localDep := &chart.Dependency{ + Name: local.Name(), + Repository: "", + Version: local.Metadata.Version, + } + + // create a 'tmpcharts' directory to test #5567 + if err := os.MkdirAll(filepath.Join(chartPath, "tmpcharts"), 0755); err != nil { + t.Fatal(err) + } + if err := m.batchDownloadAll([]*chart.Dependency{signDep, localDep}); err != nil { + t.Error(err) + } + + if _, err := os.Stat(filepath.Join(chartPath, "charts", "signtest-0.1.0.tgz")); os.IsNotExist(err) { + t.Error(err) + } +} + func TestUpdateBeforeBuild(t *testing.T) { // Set up a fake repo srv, err := repotest.NewTempServerWithCleanup(t, "testdata/*.tgz*")