Implement recursive loginc on downloader.Manager

Signed-off-by: Miguel Elias dos Santos <migueleliasweb@gmail.com>

.gitignore

@@ -10,3 +10,9 @@ bin/
 vendor/
 # Ignores charts pulled for dependency build tests
 cmd/helm/testdata/testcharts/issue-7233/charts/*
+cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/charts/*
+cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/Chart.lock
+cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/charts/*
+cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/Chart.lock
+cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/charts/*
+cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/Chart.lock

cmd/helm/dependency_build.go

@@ -68,7 +68,7 @@ func newDependencyBuildCmd(cfg *action.Configuration, out io.Writer) *cobra.Comm
 			if client.Verify {
 				man.Verify = downloader.VerifyIfPossible
 			}
-			err := man.Build()
+			err := man.Build(client.BuildOrUpdateRecursive)
 			if e, ok := err.(downloader.ErrRepoNotFound); ok {
 				return fmt.Errorf("%s. Please add the missing repos via 'helm repo add'", e.Error())
 			}
@@ -80,6 +80,7 @@ func newDependencyBuildCmd(cfg *action.Configuration, out io.Writer) *cobra.Comm
 	f.BoolVar(&client.Verify, "verify", false, "verify the packages against signatures")
 	f.StringVar(&client.Keyring, "keyring", defaultKeyring(), "keyring containing public keys")
 	f.BoolVar(&client.SkipRefresh, "skip-refresh", false, "do not refresh the local repository cache")
+	f.BoolVar(&client.BuildOrUpdateRecursive, "recursive", false, "build dependencies recursively")
 
 	return cmd
 }

cmd/helm/dependency_update.go

@@ -71,7 +71,7 @@ func newDependencyUpdateCmd(cfg *action.Configuration, out io.Writer) *cobra.Com
 			if client.Verify {
 				man.Verify = downloader.VerifyAlways
 			}
-			return man.Update()
+			return man.Update(client.BuildOrUpdateRecursive)
 		},
 	}
 
@@ -79,6 +79,7 @@ func newDependencyUpdateCmd(cfg *action.Configuration, out io.Writer) *cobra.Com
 	f.BoolVar(&client.Verify, "verify", false, "verify the packages against signatures")
 	f.StringVar(&client.Keyring, "keyring", defaultKeyring(), "keyring containing public keys")
 	f.BoolVar(&client.SkipRefresh, "skip-refresh", false, "do not refresh the local repository cache")
+	f.BoolVar(&client.BuildOrUpdateRecursive, "recursive", false, "update dependencies recursively")
 
 	return cmd
 }

cmd/helm/helm_test.go

@@ -18,9 +18,11 @@ package main
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"testing"
@@ -30,6 +32,7 @@ import (
 
 	"helm.sh/helm/v3/internal/test"
 	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/chartutil"
 	"helm.sh/helm/v3/pkg/cli"
 	kubefake "helm.sh/helm/v3/pkg/kube/fake"
@@ -58,6 +61,15 @@ func runTestCmd(t *testing.T, tests []cmdTestCase) {
 						t.Fatal(err)
 					}
 				}
+
+				if tt.preCmd != nil {
+					t.Logf("running preCmd (attempt %d): %s", i+1, tt.cmd)
+					if err := tt.preCmd(t); err != nil {
+						t.Errorf("expected no error executing preCmd, got: '%v'", err)
+						t.FailNow()
+					}
+				}
+
 				t.Logf("running cmd (attempt %d): %s", i+1, tt.cmd)
 				_, out, err := executeActionCommandC(storage, tt.cmd)
 				if tt.wantError && err == nil {
@@ -220,3 +232,43 @@ func TestPluginExitCode(t *testing.T) {
 		}
 	}
 }
+
+// resetChartDependencyState completely resets dependency state of a given chart
+// by deleting `Chart.lock` and `charts/`.
+//
+// If `recursive` is set to true, it will recurse into all local dependency charts
+// and do the same.
+func resetChartDependencyState(chartPath string, recursive bool) error {
+	chartRequested, err := loader.Load(chartPath)
+
+	if err != nil {
+		return err
+	}
+
+	os.Remove(fmt.Sprintf("%s/Chart.lock", chartPath))
+	os.RemoveAll(fmt.Sprintf("%s/charts/", chartPath))
+
+	if recursive {
+		for _, chartDep := range chartRequested.Metadata.Dependencies {
+			if strings.HasPrefix(
+				chartDep.Repository,
+				"file://",
+			) {
+
+				fullDepPath, err := filepath.Abs(
+					fmt.Sprintf("%s/%s", chartPath, chartDep.Repository[7:]),
+				)
+
+				if err != nil {
+					return err
+				}
+
+				if err := resetChartDependencyState(fullDepPath, recursive); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}

cmd/helm/install.go

@@ -172,6 +172,7 @@ func addInstallFlags(cmd *cobra.Command, f *pflag.FlagSet, client *action.Instal
 	f.StringVar(&client.Description, "description", "", "add a custom description")
 	f.BoolVar(&client.Devel, "devel", false, "use development versions, too. Equivalent to version '>0.0.0-0'. If --version is set, this is ignored")
 	f.BoolVar(&client.DependencyUpdate, "dependency-update", false, "update dependencies if they are missing before installing the chart")
+	f.BoolVar(&client.DependencyUpdateRecursive, "dependency-update-recursive", false, "update dependencies recursively if they are missing before installing the chart")
 	f.BoolVar(&client.DisableOpenAPIValidation, "disable-openapi-validation", false, "if set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema")
 	f.BoolVar(&client.Atomic, "atomic", false, "if set, the installation process deletes the installation on failure. The --wait flag will be set automatically if --atomic is used")
 	f.BoolVar(&client.SkipCRDs, "skip-crds", false, "if set, no CRDs will be installed. By default, CRDs are installed if not already present")
@@ -242,7 +243,7 @@ func runInstall(args []string, client *action.Install, valueOpts *values.Options
 		// https://github.com/helm/helm/issues/2209
 		if err := action.CheckDependencies(chartRequested, req); err != nil {
 			err = errors.Wrap(err, "An error occurred while checking for chart dependencies. You may need to run `helm dependency build` to fetch missing dependencies")
-			if client.DependencyUpdate {
+			if client.DependencyUpdate || client.DependencyUpdateRecursive {
 				man := &downloader.Manager{
 					Out:              out,
 					ChartPath:        cp,
@@ -253,7 +254,7 @@ func runInstall(args []string, client *action.Install, valueOpts *values.Options
 					RepositoryCache:  settings.RepositoryCache,
 					Debug:            settings.Debug,
 				}
-				if err := man.Update(); err != nil {
+				if err := man.Update(client.DependencyUpdateRecursive); err != nil {
 					return nil, err
 				}
 				// Reload the chart with the updated Chart.lock file.

cmd/helm/install_test.go

@@ -153,6 +153,12 @@ func TestInstall(t *testing.T) {
 			cmd:    "install --dependency-update updeps testdata/testcharts/chart-with-subchart-update",
 			golden: "output/chart-with-subchart-update.txt",
 		},
+		// Install chart with update-dependency-recursive
+		{
+			name:   "install chart with dependency update recursive",
+			cmd:    "install --dependency-update-recursive recdeps testdata/testcharts/chart-with-multi-level-deps/root/",
+			golden: "output/install-dependency-update-recursive.txt",
+		},
 		// Install, chart with bad dependencies in Chart.yaml in /charts
 		{
 			name:      "install chart with bad dependencies in Chart.yaml",

cmd/helm/package.go

@@ -96,7 +96,7 @@ func newPackageCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
 						RepositoryCache:  settings.RepositoryCache,
 					}
 
-					if err := downloadManager.Update(); err != nil {
+					if err := downloadManager.Update(client.DependencyUpdateRecursive); err != nil {
 						return err
 					}
 				}
@@ -119,6 +119,7 @@ func newPackageCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
 	f.StringVar(&client.AppVersion, "app-version", "", "set the appVersion on the chart to this version")
 	f.StringVarP(&client.Destination, "destination", "d", ".", "location to write the chart.")
 	f.BoolVarP(&client.DependencyUpdate, "dependency-update", "u", false, `update dependencies from "Chart.yaml" to dir "charts/" before packaging`)
+	f.BoolVarP(&client.DependencyUpdateRecursive, "dependency-update-recursive", "r", false, `update dependencies recursively from from "Chart.yaml" and all of its subcharts before packaging`)
 
 	return cmd
 }

cmd/helm/template_test.go

@@ -131,6 +131,22 @@ func TestTemplateCmd(t *testing.T) {
 			cmd:    fmt.Sprintf(`template '%s' --skip-tests`, chartPath),
 			golden: "output/template-skip-tests.txt",
 		},
+		{
+			name: "template with dependency update recursive",
+			preCmd: func(t *testing.T) error {
+				// We must reset the chart's dependency to actually
+				// exercise the ability to provision missing nested depencendies.
+				// If we don't do this, the chart will contain the `tgz` files from previous runs
+				// and the `--dependency-update-recursive` flag won't do much.
+				// Note the dependency files for the chart are ignored via .gitignore.
+				return resetChartDependencyState(
+					"testdata/testcharts/chart-with-multi-level-deps/root",
+					true,
+				)
+			},
+			cmd:    fmt.Sprintf(`template '%s' --dependency-update-recursive`, "testdata/testcharts/chart-with-multi-level-deps/root"),
+			golden: "output/template-dependency-update-recursive.txt",
+		},
 	}
 	runTestCmd(t, tests)
 }

cmd/helm/testdata/output/install-dependency-update-recursive.txt

@@ -0,0 +1,6 @@
+NAME: recdeps
+LAST DEPLOYED: Fri Sep  2 22:04:05 1977
+NAMESPACE: default
+STATUS: deployed
+REVISION: 1
+TEST SUITE: None

cmd/helm/testdata/output/template-dependency-update-recursive.txt

@@ -0,0 +1,22 @@
+Hang tight while we grab the latest from your chart repositories...
+...Successfully got an update from the "kube-sigs-external-dns" chart repository
+Update Complete. ⎈Happy Helming!⎈
+Saving 1 charts
+Deleting outdated charts
+Hang tight while we grab the latest from your chart repositories...
+...Successfully got an update from the "kube-sigs-external-dns" chart repository
+Update Complete. ⎈Happy Helming!⎈
+Saving 1 charts
+Deleting outdated charts
+---
+# Source: root/charts/dep1/templates/configmap1.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dep1
+---
+# Source: root/templates/configmaproot.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: root

cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/Chart.yaml

@@ -0,0 +1,9 @@
+apiVersion: v2
+name: dep1
+description: A Helm chart for Kubernetes
+type: application
+version: 0.1.0
+dependencies:
+  - name: dep2
+    repository: file://../dep2
+    version: 0.1.0

cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/templates/configmap1.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dep1

cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: dep2
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/templates/configmap2.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dep2

cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/Chart.yaml

@@ -0,0 +1,9 @@
+apiVersion: v2
+name: root
+description: A Helm chart for Kubernetes
+type: application
+version: 0.1.0
+dependencies:
+  - name: dep1
+    repository: file://../dep1
+    version: 0.1.0

cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/templates/configmaproot.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: root

cmd/helm/upgrade.go

@@ -126,6 +126,7 @@ func newUpgradeCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
 					instClient.Description = client.Description
 					instClient.DependencyUpdate = client.DependencyUpdate
                                        instClient.EnableDNS = client.EnableDNS
+					instClient.DependencyUpdateRecursive = client.DependencyUpdateRecursive
 
 					rel, err := runInstall(args, instClient, valueOpts, out)
 					if err != nil {
@@ -172,7 +173,7 @@ func newUpgradeCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
 							RepositoryCache:  settings.RepositoryCache,
 							Debug:            settings.Debug,
 						}
-						if err := man.Update(); err != nil {
+						if err := man.Update(client.DependencyUpdateRecursive); err != nil {
 							return err
 						}
 						// Reload the chart with the updated Chart.lock file.

pkg/action/dependency.go

@@ -38,6 +38,7 @@ type Dependency struct {
 	Keyring                string
 	SkipRefresh            bool
 	ColumnWidth            uint
+	BuildOrUpdateRecursive bool
 }
 
 // NewDependency creates a new Dependency object with the given configuration.

pkg/action/install.go

@@ -79,6 +79,7 @@ type Install struct {
 	WaitForJobs               bool
 	Devel                     bool
 	DependencyUpdate          bool
+	DependencyUpdateRecursive bool
 	Timeout                   time.Duration
 	Namespace                 string
 	ReleaseName               string

pkg/action/package.go

@@ -43,6 +43,7 @@ type Package struct {
 	AppVersion                string
 	Destination               string
 	DependencyUpdate          bool
+	DependencyUpdateRecursive bool
 
 	RepositoryConfig string
 	RepositoryCache  string

pkg/action/upgrade.go

@@ -102,6 +102,8 @@ type Upgrade struct {
 	DisableOpenAPIValidation bool
 	// Get missing dependencies
 	DependencyUpdate bool
+	// Get missing dependencies, recursively
+	DependencyUpdateRecursive bool
 	// Lock to control raceconditions when the process receives a SIGTERM
 	Lock sync.Mutex
 	// Enable DNS lookups when rendering templates

pkg/downloader/manager.go

@@ -82,8 +82,8 @@ type Manager struct {
 // If the lockfile is not present, this will run a Manager.Update()
 //
 // If SkipUpdate is set, this will not update the repository.
-func (m *Manager) Build() error {
-	c, err := m.loadChartDir()
+func (m *Manager) Build(recursive bool) error {
+	c, err := m.loadChartDir(m.ChartPath)
 	if err != nil {
 		return err
 	}
@@ -92,7 +92,7 @@ func (m *Manager) Build() error {
 	// an update.
 	lock := c.Lock
 	if lock == nil {
-		return m.Update()
+		return m.Update(recursive)
 	}
 
 	// Check that all of the repos we're dependent on actually exist.
@@ -149,8 +149,30 @@ func (m *Manager) Build() error {
 // It first reads the Chart.yaml file, and then attempts to
 // negotiate versions based on that. It will download the versions
 // from remote chart repositories unless SkipUpdate is true.
-func (m *Manager) Update() error {
-	c, err := m.loadChartDir()
+//
+// If `recursive` is set to true, it will iterate over all dependencies
+// recursively and perform the update.
+func (m *Manager) Update(recursive bool) error {
+	if recursive {
+		depChartPaths, err := m.locateDependencies(m.ChartPath, recursive)
+
+		if err != nil {
+			return err
+		}
+
+		for _, depChartPath := range depChartPaths {
+			if err := m.doUpdate(depChartPath); err != nil {
+				return err
+			}
+		}
+	}
+
+	// for last, update the root chart
+	return m.doUpdate(m.ChartPath)
+}
+
+func (m *Manager) doUpdate(chartPath string) error {
+	c, err := m.loadChartDir(chartPath)
 	if err != nil {
 		return err
 	}
@@ -218,13 +240,13 @@ func (m *Manager) Update() error {
 	return writeLock(m.ChartPath, lock, c.Metadata.APIVersion == chart.APIVersionV1)
 }
 
-func (m *Manager) loadChartDir() (*chart.Chart, error) {
-	if fi, err := os.Stat(m.ChartPath); err != nil {
-		return nil, errors.Wrapf(err, "could not find %s", m.ChartPath)
+func (m *Manager) loadChartDir(chartPath string) (*chart.Chart, error) {
+	if fi, err := os.Stat(chartPath); err != nil {
+		return nil, errors.Wrapf(err, "could not find %s", chartPath)
 	} else if !fi.IsDir() {
 		return nil, errors.New("only unpacked charts can be updated")
 	}
-	return loader.LoadDir(m.ChartPath)
+	return loader.LoadDir(chartPath)
 }
 
 // resolve takes a list of dependencies and translates them into an exact version to download.
@@ -742,6 +764,63 @@ func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*
 	return url, username, password, false, false, "", "", "", err
 }
 
+// LocateDependencies locates the dependencies for the given chart (optionally recursively)
+//
+// The returned list of Chart paths is ordered from "leaf to root" so we can issue updates in
+// the right order when iterating over this list.
+func (m *Manager) locateDependencies(baseChartPath string, resursive bool) ([]string, error) {
+	reversedDeps := []string{}
+
+	baseChart, err := m.loadChartDir(baseChartPath)
+
+	if err != nil {
+		return nil, err
+	}
+
+	for _, chartDependency := range baseChart.Metadata.Dependencies {
+
+		fullDepChartPath := chartDependency.Repository
+		isLocalChart := false
+
+		if strings.HasPrefix(
+			chartDependency.Repository,
+			"file://",
+		) {
+			isLocalChart = true
+
+			fullDepChartPath, err = filepath.Abs(
+				fmt.Sprintf(
+					"%s/%s",
+					baseChartPath, chartDependency.Repository[7:]), // removes "file://"
+			)
+
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		reversedDeps = append(
+			[]string{fullDepChartPath},
+			reversedDeps...,
+		)
+
+		if resursive && isLocalChart {
+			subDeps, err := m.locateDependencies(fullDepChartPath, resursive)
+
+			if err != nil {
+				return nil, err
+			}
+
+			reversedDeps = append(
+				subDeps,
+				reversedDeps...,
+			)
+		}
+	}
+
+	return reversedDeps, nil
+}
+
 // findEntryByName finds an entry in the chart repository whose name matches the given name.
 //
 // It returns the ChartVersions for that entry.

pkg/downloader/manager_test.go

@@ -321,12 +321,12 @@ func TestUpdateBeforeBuild(t *testing.T) {
 	}
 
 	// Update before Build. see issue: https://github.com/helm/helm/issues/7101
-	err = m.Update()
+	err = m.Update(false)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	err = m.Build()
+	err = m.Build(false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -396,7 +396,7 @@ func TestUpdateWithNoRepo(t *testing.T) {
 	}
 
 	// Test the update
-	err = m.Update()
+	err = m.Update(false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -461,13 +461,13 @@ func checkBuildWithOptionalFields(t *testing.T, chartName string, dep chart.Depe
 	}
 
 	// First build will update dependencies and create Chart.lock file.
-	err = m.Build()
+	err = m.Build(false)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Second build should be passed. See PR #6655.
-	err = m.Build()
+	err = m.Build(false)
 	if err != nil {
 		t.Fatal(err)
 	}