From 271e9231bcc342c4d7d1def184dba644928ea237 Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Thu, 26 Jan 2023 00:26:41 +1100 Subject: [PATCH 1/9] Implement recursive loginc on downloader.Manager Signed-off-by: Miguel Elias dos Santos --- .gitignore | 6 ++ cmd/helm/dependency_build.go | 3 +- cmd/helm/dependency_update.go | 3 +- cmd/helm/helm_test.go | 52 ++++++++++ cmd/helm/install.go | 5 +- cmd/helm/install_test.go | 6 ++ cmd/helm/package.go | 3 +- cmd/helm/template_test.go | 16 +++ .../install-dependency-update-recursive.txt | 6 ++ .../template-dependency-update-recursive.txt | 22 +++++ .../dep1/Chart.yaml | 9 ++ .../dep1/templates/configmap1.yaml | 4 + .../dep1/values.yaml | 0 .../dep2/Chart.yaml | 24 +++++ .../dep2/templates/configmap2.yaml | 4 + .../dep2/values.yaml | 0 .../root/Chart.yaml | 9 ++ .../root/templates/configmaproot.yaml | 4 + .../root/values.yaml | 0 cmd/helm/upgrade.go | 5 +- pkg/action/dependency.go | 9 +- pkg/action/install.go | 45 ++++----- pkg/action/package.go | 17 ++-- pkg/action/upgrade.go | 2 + pkg/downloader/manager.go | 97 +++++++++++++++++-- pkg/downloader/manager_test.go | 10 +- 26 files changed, 306 insertions(+), 55 deletions(-) create mode 100644 cmd/helm/testdata/output/install-dependency-update-recursive.txt create mode 100644 cmd/helm/testdata/output/template-dependency-update-recursive.txt create mode 100644 cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/Chart.yaml create mode 100644 cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/templates/configmap1.yaml create mode 100644 cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/values.yaml create mode 100644 cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/Chart.yaml create mode 100644 cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/templates/configmap2.yaml create mode 100644 cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/values.yaml create mode 100644 cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/Chart.yaml create mode 100644 cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/templates/configmaproot.yaml create mode 100644 cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/values.yaml diff --git a/.gitignore b/.gitignore index d1af995a3..3121d4e69 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,9 @@ bin/ vendor/ # Ignores charts pulled for dependency build tests cmd/helm/testdata/testcharts/issue-7233/charts/* +cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/charts/* +cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/Chart.lock +cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/charts/* +cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/Chart.lock +cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/charts/* +cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/Chart.lock \ No newline at end of file diff --git a/cmd/helm/dependency_build.go b/cmd/helm/dependency_build.go index 1ee46d3d2..4985be9a4 100644 --- a/cmd/helm/dependency_build.go +++ b/cmd/helm/dependency_build.go @@ -68,7 +68,7 @@ func newDependencyBuildCmd(cfg *action.Configuration, out io.Writer) *cobra.Comm if client.Verify { man.Verify = downloader.VerifyIfPossible } - err := man.Build() + err := man.Build(client.BuildOrUpdateRecursive) if e, ok := err.(downloader.ErrRepoNotFound); ok { return fmt.Errorf("%s. Please add the missing repos via 'helm repo add'", e.Error()) } @@ -80,6 +80,7 @@ func newDependencyBuildCmd(cfg *action.Configuration, out io.Writer) *cobra.Comm f.BoolVar(&client.Verify, "verify", false, "verify the packages against signatures") f.StringVar(&client.Keyring, "keyring", defaultKeyring(), "keyring containing public keys") f.BoolVar(&client.SkipRefresh, "skip-refresh", false, "do not refresh the local repository cache") + f.BoolVar(&client.BuildOrUpdateRecursive, "recursive", false, "build dependencies recursively") return cmd } diff --git a/cmd/helm/dependency_update.go b/cmd/helm/dependency_update.go index ad0188f17..0105d3816 100644 --- a/cmd/helm/dependency_update.go +++ b/cmd/helm/dependency_update.go @@ -71,7 +71,7 @@ func newDependencyUpdateCmd(cfg *action.Configuration, out io.Writer) *cobra.Com if client.Verify { man.Verify = downloader.VerifyAlways } - return man.Update() + return man.Update(client.BuildOrUpdateRecursive) }, } @@ -79,6 +79,7 @@ func newDependencyUpdateCmd(cfg *action.Configuration, out io.Writer) *cobra.Com f.BoolVar(&client.Verify, "verify", false, "verify the packages against signatures") f.StringVar(&client.Keyring, "keyring", defaultKeyring(), "keyring containing public keys") f.BoolVar(&client.SkipRefresh, "skip-refresh", false, "do not refresh the local repository cache") + f.BoolVar(&client.BuildOrUpdateRecursive, "recursive", false, "update dependencies recursively") return cmd } diff --git a/cmd/helm/helm_test.go b/cmd/helm/helm_test.go index b20b1a24d..48e442588 100644 --- a/cmd/helm/helm_test.go +++ b/cmd/helm/helm_test.go @@ -18,9 +18,11 @@ package main import ( "bytes" + "fmt" "io" "os" "os/exec" + "path/filepath" "runtime" "strings" "testing" @@ -30,6 +32,7 @@ import ( "helm.sh/helm/v3/internal/test" "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" "helm.sh/helm/v3/pkg/cli" kubefake "helm.sh/helm/v3/pkg/kube/fake" @@ -58,6 +61,15 @@ func runTestCmd(t *testing.T, tests []cmdTestCase) { t.Fatal(err) } } + + if tt.preCmd != nil { + t.Logf("running preCmd (attempt %d): %s", i+1, tt.cmd) + if err := tt.preCmd(t); err != nil { + t.Errorf("expected no error executing preCmd, got: '%v'", err) + t.FailNow() + } + } + t.Logf("running cmd (attempt %d): %s", i+1, tt.cmd) _, out, err := executeActionCommandC(storage, tt.cmd) if tt.wantError && err == nil { @@ -220,3 +232,43 @@ func TestPluginExitCode(t *testing.T) { } } } + +// resetChartDependencyState completely resets dependency state of a given chart +// by deleting `Chart.lock` and `charts/`. +// +// If `recursive` is set to true, it will recurse into all local dependency charts +// and do the same. +func resetChartDependencyState(chartPath string, recursive bool) error { + chartRequested, err := loader.Load(chartPath) + + if err != nil { + return err + } + + os.Remove(fmt.Sprintf("%s/Chart.lock", chartPath)) + os.RemoveAll(fmt.Sprintf("%s/charts/", chartPath)) + + if recursive { + for _, chartDep := range chartRequested.Metadata.Dependencies { + if strings.HasPrefix( + chartDep.Repository, + "file://", + ) { + + fullDepPath, err := filepath.Abs( + fmt.Sprintf("%s/%s", chartPath, chartDep.Repository[7:]), + ) + + if err != nil { + return err + } + + if err := resetChartDependencyState(fullDepPath, recursive); err != nil { + return err + } + } + } + } + + return nil +} diff --git a/cmd/helm/install.go b/cmd/helm/install.go index 13c674066..84f93fb42 100644 --- a/cmd/helm/install.go +++ b/cmd/helm/install.go @@ -172,6 +172,7 @@ func addInstallFlags(cmd *cobra.Command, f *pflag.FlagSet, client *action.Instal f.StringVar(&client.Description, "description", "", "add a custom description") f.BoolVar(&client.Devel, "devel", false, "use development versions, too. Equivalent to version '>0.0.0-0'. If --version is set, this is ignored") f.BoolVar(&client.DependencyUpdate, "dependency-update", false, "update dependencies if they are missing before installing the chart") + f.BoolVar(&client.DependencyUpdateRecursive, "dependency-update-recursive", false, "update dependencies recursively if they are missing before installing the chart") f.BoolVar(&client.DisableOpenAPIValidation, "disable-openapi-validation", false, "if set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema") f.BoolVar(&client.Atomic, "atomic", false, "if set, the installation process deletes the installation on failure. The --wait flag will be set automatically if --atomic is used") f.BoolVar(&client.SkipCRDs, "skip-crds", false, "if set, no CRDs will be installed. By default, CRDs are installed if not already present") @@ -242,7 +243,7 @@ func runInstall(args []string, client *action.Install, valueOpts *values.Options // https://github.com/helm/helm/issues/2209 if err := action.CheckDependencies(chartRequested, req); err != nil { err = errors.Wrap(err, "An error occurred while checking for chart dependencies. You may need to run `helm dependency build` to fetch missing dependencies") - if client.DependencyUpdate { + if client.DependencyUpdate || client.DependencyUpdateRecursive { man := &downloader.Manager{ Out: out, ChartPath: cp, @@ -253,7 +254,7 @@ func runInstall(args []string, client *action.Install, valueOpts *values.Options RepositoryCache: settings.RepositoryCache, Debug: settings.Debug, } - if err := man.Update(); err != nil { + if err := man.Update(client.DependencyUpdateRecursive); err != nil { return nil, err } // Reload the chart with the updated Chart.lock file. diff --git a/cmd/helm/install_test.go b/cmd/helm/install_test.go index b34d1455c..91490b026 100644 --- a/cmd/helm/install_test.go +++ b/cmd/helm/install_test.go @@ -153,6 +153,12 @@ func TestInstall(t *testing.T) { cmd: "install --dependency-update updeps testdata/testcharts/chart-with-subchart-update", golden: "output/chart-with-subchart-update.txt", }, + // Install chart with update-dependency-recursive + { + name: "install chart with dependency update recursive", + cmd: "install --dependency-update-recursive recdeps testdata/testcharts/chart-with-multi-level-deps/root/", + golden: "output/install-dependency-update-recursive.txt", + }, // Install, chart with bad dependencies in Chart.yaml in /charts { name: "install chart with bad dependencies in Chart.yaml", diff --git a/cmd/helm/package.go b/cmd/helm/package.go index 822d3d56a..c344df9d9 100644 --- a/cmd/helm/package.go +++ b/cmd/helm/package.go @@ -96,7 +96,7 @@ func newPackageCmd(cfg *action.Configuration, out io.Writer) *cobra.Command { RepositoryCache: settings.RepositoryCache, } - if err := downloadManager.Update(); err != nil { + if err := downloadManager.Update(client.DependencyUpdateRecursive); err != nil { return err } } @@ -119,6 +119,7 @@ func newPackageCmd(cfg *action.Configuration, out io.Writer) *cobra.Command { f.StringVar(&client.AppVersion, "app-version", "", "set the appVersion on the chart to this version") f.StringVarP(&client.Destination, "destination", "d", ".", "location to write the chart.") f.BoolVarP(&client.DependencyUpdate, "dependency-update", "u", false, `update dependencies from "Chart.yaml" to dir "charts/" before packaging`) + f.BoolVarP(&client.DependencyUpdateRecursive, "dependency-update-recursive", "r", false, `update dependencies recursively from from "Chart.yaml" and all of its subcharts before packaging`) return cmd } diff --git a/cmd/helm/template_test.go b/cmd/helm/template_test.go index d1f17fe98..7c1235052 100644 --- a/cmd/helm/template_test.go +++ b/cmd/helm/template_test.go @@ -131,6 +131,22 @@ func TestTemplateCmd(t *testing.T) { cmd: fmt.Sprintf(`template '%s' --skip-tests`, chartPath), golden: "output/template-skip-tests.txt", }, + { + name: "template with dependency update recursive", + preCmd: func(t *testing.T) error { + // We must reset the chart's dependency to actually + // exercise the ability to provision missing nested depencendies. + // If we don't do this, the chart will contain the `tgz` files from previous runs + // and the `--dependency-update-recursive` flag won't do much. + // Note the dependency files for the chart are ignored via .gitignore. + return resetChartDependencyState( + "testdata/testcharts/chart-with-multi-level-deps/root", + true, + ) + }, + cmd: fmt.Sprintf(`template '%s' --dependency-update-recursive`, "testdata/testcharts/chart-with-multi-level-deps/root"), + golden: "output/template-dependency-update-recursive.txt", + }, } runTestCmd(t, tests) } diff --git a/cmd/helm/testdata/output/install-dependency-update-recursive.txt b/cmd/helm/testdata/output/install-dependency-update-recursive.txt new file mode 100644 index 000000000..9b25fbe36 --- /dev/null +++ b/cmd/helm/testdata/output/install-dependency-update-recursive.txt @@ -0,0 +1,6 @@ +NAME: recdeps +LAST DEPLOYED: Fri Sep 2 22:04:05 1977 +NAMESPACE: default +STATUS: deployed +REVISION: 1 +TEST SUITE: None diff --git a/cmd/helm/testdata/output/template-dependency-update-recursive.txt b/cmd/helm/testdata/output/template-dependency-update-recursive.txt new file mode 100644 index 000000000..46a53092f --- /dev/null +++ b/cmd/helm/testdata/output/template-dependency-update-recursive.txt @@ -0,0 +1,22 @@ +Hang tight while we grab the latest from your chart repositories... +...Successfully got an update from the "kube-sigs-external-dns" chart repository +Update Complete. ⎈Happy Helming!⎈ +Saving 1 charts +Deleting outdated charts +Hang tight while we grab the latest from your chart repositories... +...Successfully got an update from the "kube-sigs-external-dns" chart repository +Update Complete. ⎈Happy Helming!⎈ +Saving 1 charts +Deleting outdated charts +--- +# Source: root/charts/dep1/templates/configmap1.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: dep1 +--- +# Source: root/templates/configmaproot.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: root diff --git a/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/Chart.yaml b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/Chart.yaml new file mode 100644 index 000000000..2919254ca --- /dev/null +++ b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: dep1 +description: A Helm chart for Kubernetes +type: application +version: 0.1.0 +dependencies: + - name: dep2 + repository: file://../dep2 + version: 0.1.0 \ No newline at end of file diff --git a/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/templates/configmap1.yaml b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/templates/configmap1.yaml new file mode 100644 index 000000000..e80109f69 --- /dev/null +++ b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/templates/configmap1.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: dep1 diff --git a/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/values.yaml b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep1/values.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/Chart.yaml b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/Chart.yaml new file mode 100644 index 000000000..ec045301e --- /dev/null +++ b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: dep2 +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/templates/configmap2.yaml b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/templates/configmap2.yaml new file mode 100644 index 000000000..899e7f6ba --- /dev/null +++ b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/templates/configmap2.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: dep2 diff --git a/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/values.yaml b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/dep2/values.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/Chart.yaml b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/Chart.yaml new file mode 100644 index 000000000..241e9ed0e --- /dev/null +++ b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: root +description: A Helm chart for Kubernetes +type: application +version: 0.1.0 +dependencies: + - name: dep1 + repository: file://../dep1 + version: 0.1.0 \ No newline at end of file diff --git a/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/templates/configmaproot.yaml b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/templates/configmaproot.yaml new file mode 100644 index 000000000..2caac7cce --- /dev/null +++ b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/templates/configmaproot.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: root diff --git a/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/values.yaml b/cmd/helm/testdata/testcharts/chart-with-multi-level-deps/root/values.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/helm/upgrade.go b/cmd/helm/upgrade.go index 145d342b7..f40f8a16a 100644 --- a/cmd/helm/upgrade.go +++ b/cmd/helm/upgrade.go @@ -125,7 +125,8 @@ func newUpgradeCmd(cfg *action.Configuration, out io.Writer) *cobra.Command { instClient.SubNotes = client.SubNotes instClient.Description = client.Description instClient.DependencyUpdate = client.DependencyUpdate - instClient.EnableDNS = client.EnableDNS + instClient.EnableDNS = client.EnableDNS + instClient.DependencyUpdateRecursive = client.DependencyUpdateRecursive rel, err := runInstall(args, instClient, valueOpts, out) if err != nil { @@ -172,7 +173,7 @@ func newUpgradeCmd(cfg *action.Configuration, out io.Writer) *cobra.Command { RepositoryCache: settings.RepositoryCache, Debug: settings.Debug, } - if err := man.Update(); err != nil { + if err := man.Update(client.DependencyUpdateRecursive); err != nil { return err } // Reload the chart with the updated Chart.lock file. diff --git a/pkg/action/dependency.go b/pkg/action/dependency.go index 3265f1f17..849c99b6a 100644 --- a/pkg/action/dependency.go +++ b/pkg/action/dependency.go @@ -34,10 +34,11 @@ import ( // // It provides the implementation of 'helm dependency' and its respective subcommands. type Dependency struct { - Verify bool - Keyring string - SkipRefresh bool - ColumnWidth uint + Verify bool + Keyring string + SkipRefresh bool + ColumnWidth uint + BuildOrUpdateRecursive bool } // NewDependency creates a new Dependency object with the given configuration. diff --git a/pkg/action/install.go b/pkg/action/install.go index d5c34cef7..3262ad6f4 100644 --- a/pkg/action/install.go +++ b/pkg/action/install.go @@ -69,28 +69,29 @@ type Install struct { ChartPathOptions - ClientOnly bool - Force bool - CreateNamespace bool - DryRun bool - DisableHooks bool - Replace bool - Wait bool - WaitForJobs bool - Devel bool - DependencyUpdate bool - Timeout time.Duration - Namespace string - ReleaseName string - GenerateName bool - NameTemplate string - Description string - OutputDir string - Atomic bool - SkipCRDs bool - SubNotes bool - DisableOpenAPIValidation bool - IncludeCRDs bool + ClientOnly bool + Force bool + CreateNamespace bool + DryRun bool + DisableHooks bool + Replace bool + Wait bool + WaitForJobs bool + Devel bool + DependencyUpdate bool + DependencyUpdateRecursive bool + Timeout time.Duration + Namespace string + ReleaseName string + GenerateName bool + NameTemplate string + Description string + OutputDir string + Atomic bool + SkipCRDs bool + SubNotes bool + DisableOpenAPIValidation bool + IncludeCRDs bool // KubeVersion allows specifying a custom kubernetes version to use and // APIVersions allows a manual set of supported API Versions to be passed // (for things like templating). These are ignored if ClientOnly is false diff --git a/pkg/action/package.go b/pkg/action/package.go index 698169032..d954510e1 100644 --- a/pkg/action/package.go +++ b/pkg/action/package.go @@ -35,14 +35,15 @@ import ( // // It provides the implementation of 'helm package'. type Package struct { - Sign bool - Key string - Keyring string - PassphraseFile string - Version string - AppVersion string - Destination string - DependencyUpdate bool + Sign bool + Key string + Keyring string + PassphraseFile string + Version string + AppVersion string + Destination string + DependencyUpdate bool + DependencyUpdateRecursive bool RepositoryConfig string RepositoryCache string diff --git a/pkg/action/upgrade.go b/pkg/action/upgrade.go index 829be51df..c183b4c05 100644 --- a/pkg/action/upgrade.go +++ b/pkg/action/upgrade.go @@ -102,6 +102,8 @@ type Upgrade struct { DisableOpenAPIValidation bool // Get missing dependencies DependencyUpdate bool + // Get missing dependencies, recursively + DependencyUpdateRecursive bool // Lock to control raceconditions when the process receives a SIGTERM Lock sync.Mutex // Enable DNS lookups when rendering templates diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index 9de33a166..a0af580d5 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -82,8 +82,8 @@ type Manager struct { // If the lockfile is not present, this will run a Manager.Update() // // If SkipUpdate is set, this will not update the repository. -func (m *Manager) Build() error { - c, err := m.loadChartDir() +func (m *Manager) Build(recursive bool) error { + c, err := m.loadChartDir(m.ChartPath) if err != nil { return err } @@ -92,7 +92,7 @@ func (m *Manager) Build() error { // an update. lock := c.Lock if lock == nil { - return m.Update() + return m.Update(recursive) } // Check that all of the repos we're dependent on actually exist. @@ -149,8 +149,30 @@ func (m *Manager) Build() error { // It first reads the Chart.yaml file, and then attempts to // negotiate versions based on that. It will download the versions // from remote chart repositories unless SkipUpdate is true. -func (m *Manager) Update() error { - c, err := m.loadChartDir() +// +// If `recursive` is set to true, it will iterate over all dependencies +// recursively and perform the update. +func (m *Manager) Update(recursive bool) error { + if recursive { + depChartPaths, err := m.locateDependencies(m.ChartPath, recursive) + + if err != nil { + return err + } + + for _, depChartPath := range depChartPaths { + if err := m.doUpdate(depChartPath); err != nil { + return err + } + } + } + + // for last, update the root chart + return m.doUpdate(m.ChartPath) +} + +func (m *Manager) doUpdate(chartPath string) error { + c, err := m.loadChartDir(chartPath) if err != nil { return err } @@ -218,13 +240,13 @@ func (m *Manager) Update() error { return writeLock(m.ChartPath, lock, c.Metadata.APIVersion == chart.APIVersionV1) } -func (m *Manager) loadChartDir() (*chart.Chart, error) { - if fi, err := os.Stat(m.ChartPath); err != nil { - return nil, errors.Wrapf(err, "could not find %s", m.ChartPath) +func (m *Manager) loadChartDir(chartPath string) (*chart.Chart, error) { + if fi, err := os.Stat(chartPath); err != nil { + return nil, errors.Wrapf(err, "could not find %s", chartPath) } else if !fi.IsDir() { return nil, errors.New("only unpacked charts can be updated") } - return loader.LoadDir(m.ChartPath) + return loader.LoadDir(chartPath) } // resolve takes a list of dependencies and translates them into an exact version to download. @@ -742,6 +764,63 @@ func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]* return url, username, password, false, false, "", "", "", err } +// LocateDependencies locates the dependencies for the given chart (optionally recursively) +// +// The returned list of Chart paths is ordered from "leaf to root" so we can issue updates in +// the right order when iterating over this list. +func (m *Manager) locateDependencies(baseChartPath string, resursive bool) ([]string, error) { + reversedDeps := []string{} + + baseChart, err := m.loadChartDir(baseChartPath) + + if err != nil { + return nil, err + } + + for _, chartDependency := range baseChart.Metadata.Dependencies { + + fullDepChartPath := chartDependency.Repository + isLocalChart := false + + if strings.HasPrefix( + chartDependency.Repository, + "file://", + ) { + isLocalChart = true + + fullDepChartPath, err = filepath.Abs( + fmt.Sprintf( + "%s/%s", + baseChartPath, chartDependency.Repository[7:]), // removes "file://" + ) + + if err != nil { + return nil, err + } + } + + reversedDeps = append( + []string{fullDepChartPath}, + reversedDeps..., + ) + + if resursive && isLocalChart { + subDeps, err := m.locateDependencies(fullDepChartPath, resursive) + + if err != nil { + return nil, err + } + + reversedDeps = append( + subDeps, + reversedDeps..., + ) + } + } + + return reversedDeps, nil +} + // findEntryByName finds an entry in the chart repository whose name matches the given name. // // It returns the ChartVersions for that entry. diff --git a/pkg/downloader/manager_test.go b/pkg/downloader/manager_test.go index f7ab1a568..72a2e3575 100644 --- a/pkg/downloader/manager_test.go +++ b/pkg/downloader/manager_test.go @@ -321,12 +321,12 @@ func TestUpdateBeforeBuild(t *testing.T) { } // Update before Build. see issue: https://github.com/helm/helm/issues/7101 - err = m.Update() + err = m.Update(false) if err != nil { t.Fatal(err) } - err = m.Build() + err = m.Build(false) if err != nil { t.Fatal(err) } @@ -396,7 +396,7 @@ func TestUpdateWithNoRepo(t *testing.T) { } // Test the update - err = m.Update() + err = m.Update(false) if err != nil { t.Fatal(err) } @@ -461,13 +461,13 @@ func checkBuildWithOptionalFields(t *testing.T, chartName string, dep chart.Depe } // First build will update dependencies and create Chart.lock file. - err = m.Build() + err = m.Build(false) if err != nil { t.Fatal(err) } // Second build should be passed. See PR #6655. - err = m.Build() + err = m.Build(false) if err != nil { t.Fatal(err) } From 46d3d8c696593ceb5d129402cb0ffa2e5da71727 Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Wed, 8 Feb 2023 09:35:14 +1100 Subject: [PATCH 2/9] Rename locateDependencies to locateLocalDependencies Signed-off-by: Miguel Elias dos Santos --- pkg/downloader/manager.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index a0af580d5..1d7231ce9 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -154,7 +154,7 @@ func (m *Manager) Build(recursive bool) error { // recursively and perform the update. func (m *Manager) Update(recursive bool) error { if recursive { - depChartPaths, err := m.locateDependencies(m.ChartPath, recursive) + depChartPaths, err := m.locateLocalDependencies(m.ChartPath, recursive) if err != nil { return err @@ -764,11 +764,11 @@ func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]* return url, username, password, false, false, "", "", "", err } -// LocateDependencies locates the dependencies for the given chart (optionally recursively) +// locateLocalDependencies locates local dependencies for the given chart (optionally recursively) // // The returned list of Chart paths is ordered from "leaf to root" so we can issue updates in // the right order when iterating over this list. -func (m *Manager) locateDependencies(baseChartPath string, resursive bool) ([]string, error) { +func (m *Manager) locateLocalDependencies(baseChartPath string, resursive bool) ([]string, error) { reversedDeps := []string{} baseChart, err := m.loadChartDir(baseChartPath) @@ -780,13 +780,11 @@ func (m *Manager) locateDependencies(baseChartPath string, resursive bool) ([]st for _, chartDependency := range baseChart.Metadata.Dependencies { fullDepChartPath := chartDependency.Repository - isLocalChart := false if strings.HasPrefix( chartDependency.Repository, "file://", ) { - isLocalChart = true fullDepChartPath, err = filepath.Abs( fmt.Sprintf( @@ -797,15 +795,15 @@ func (m *Manager) locateDependencies(baseChartPath string, resursive bool) ([]st if err != nil { return nil, err } - } - reversedDeps = append( - []string{fullDepChartPath}, - reversedDeps..., - ) + reversedDeps = append( + []string{fullDepChartPath}, + reversedDeps..., + ) + } - if resursive && isLocalChart { - subDeps, err := m.locateDependencies(fullDepChartPath, resursive) + if resursive { + subDeps, err := m.locateLocalDependencies(fullDepChartPath, resursive) if err != nil { return nil, err From f7b138ee8f03ace8b89218160421778296b23110 Mon Sep 17 00:00:00 2001 From: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> Date: Mon, 12 May 2025 22:20:59 +0200 Subject: [PATCH 3/9] refactor and fix conflicts Signed-off-by: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> --- cmd/helm/helm_test.go | 2 +- pkg/action/dependency.go | 2 +- pkg/action/install.go | 19 +++++++++---------- pkg/cmd/dependency_build.go | 2 +- pkg/cmd/template_test.go | 2 +- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cmd/helm/helm_test.go b/cmd/helm/helm_test.go index bb4cd8cd1..95d4d1869 100644 --- a/cmd/helm/helm_test.go +++ b/cmd/helm/helm_test.go @@ -26,7 +26,7 @@ import ( func TestPluginExitCode(t *testing.T) { if os.Getenv("RUN_MAIN_FOR_TESTING") == "1" { - + os.Args = []string{"helm", "exitwith", "2"} // We DO call helm's main() here. So this looks like a normal `helm` process. main() diff --git a/pkg/action/dependency.go b/pkg/action/dependency.go index 5dc2b9bcd..1d16bcda9 100644 --- a/pkg/action/dependency.go +++ b/pkg/action/dependency.go @@ -38,7 +38,6 @@ type Dependency struct { Keyring string SkipRefresh bool ColumnWidth uint - BuildOrUpdateRecursive bool Username string Password string CertFile string @@ -46,6 +45,7 @@ type Dependency struct { CaFile string InsecureSkipTLSverify bool PlainHTTP bool + BuildOrUpdateRecursive bool } // NewDependency creates a new Dependency object with the given configuration. diff --git a/pkg/action/install.go b/pkg/action/install.go index 7a454ff90..2884ad72c 100644 --- a/pkg/action/install.go +++ b/pkg/action/install.go @@ -71,21 +71,17 @@ type Install struct { ChartPathOptions - DryRunOption string + ClientOnly bool + Force bool + CreateNamespace bool + DryRun bool + DryRunOption string // HideSecret can be set to true when DryRun is enabled in order to hide // Kubernetes Secrets in the output. It cannot be used outside of DryRun. HideSecret bool - WaitStrategy kube.WaitStrategy - HideNotes bool - SkipSchemaValidation bool - Labels map[string]string - ClientOnly bool - Force bool - CreateNamespace bool - DryRun bool DisableHooks bool Replace bool - Wait bool + WaitStrategy kube.WaitStrategy WaitForJobs bool Devel bool DependencyUpdate bool @@ -100,8 +96,11 @@ type Install struct { Atomic bool SkipCRDs bool SubNotes bool + HideNotes bool + SkipSchemaValidation bool DisableOpenAPIValidation bool IncludeCRDs bool + Labels map[string]string // KubeVersion allows specifying a custom kubernetes version to use and // APIVersions allows a manual set of supported API Versions to be passed // (for things like templating). These are ignored if ClientOnly is false diff --git a/pkg/cmd/dependency_build.go b/pkg/cmd/dependency_build.go index 68a76e0b8..075a26fed 100644 --- a/pkg/cmd/dependency_build.go +++ b/pkg/cmd/dependency_build.go @@ -74,7 +74,7 @@ func newDependencyBuildCmd(out io.Writer) *cobra.Command { if client.Verify { man.Verify = downloader.VerifyIfPossible } - err := man.Build(client.BuildOrUpdateRecursive) + err = man.Build(client.BuildOrUpdateRecursive) if e, ok := err.(downloader.ErrRepoNotFound); ok { return fmt.Errorf("%s. Please add the missing repos via 'helm repo add'", e.Error()) } diff --git a/pkg/cmd/template_test.go b/pkg/cmd/template_test.go index 68faa70b2..b1a3f2ab9 100644 --- a/pkg/cmd/template_test.go +++ b/pkg/cmd/template_test.go @@ -147,7 +147,7 @@ func TestTemplateCmd(t *testing.T) { }, { name: "template with dependency update recursive", - preCmd: func(t *testing.T) error { + preCmd: func(_ *testing.T) error { // We must reset the chart's dependency to actually // exercise the ability to provision missing nested depencendies. // If we don't do this, the chart will contain the `tgz` files from previous runs From 55697e89b437d61fa6a79f7b72b9616b5f69e351 Mon Sep 17 00:00:00 2001 From: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> Date: Mon, 12 May 2025 22:35:00 +0200 Subject: [PATCH 4/9] chore: revert back new line Signed-off-by: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> --- cmd/helm/helm_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/helm/helm_test.go b/cmd/helm/helm_test.go index 95d4d1869..5431daad0 100644 --- a/cmd/helm/helm_test.go +++ b/cmd/helm/helm_test.go @@ -27,6 +27,7 @@ import ( func TestPluginExitCode(t *testing.T) { if os.Getenv("RUN_MAIN_FOR_TESTING") == "1" { os.Args = []string{"helm", "exitwith", "2"} + // We DO call helm's main() here. So this looks like a normal `helm` process. main() From 6f01480d8a7a338b4b24cc63691894d7ca4898cc Mon Sep 17 00:00:00 2001 From: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> Date: Fri, 19 Sep 2025 17:37:57 +0200 Subject: [PATCH 5/9] Update pkg/downloader/manager.go Co-authored-by: Evans Mungai Signed-off-by: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> --- pkg/downloader/manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index 8a273895f..0c65550cf 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -241,6 +241,7 @@ func (m *Manager) doUpdate(chartPath string) error { } func (m *Manager) loadChartDir(chartPath string) (*chart.Chart, error) { + slog.Debug("loading chart directory", "chartPath", chartPath) if fi, err := os.Stat(chartPath); err != nil { return nil, fmt.Errorf("could not find %s: %w", chartPath, err) } else if !fi.IsDir() { From 84f260e6b53253c497b9569185f181d696fa3874 Mon Sep 17 00:00:00 2001 From: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> Date: Fri, 19 Sep 2025 17:38:15 +0200 Subject: [PATCH 6/9] Update pkg/downloader/manager.go Co-authored-by: Evans Mungai Signed-off-by: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> --- pkg/downloader/manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index 0c65550cf..9c05b5b22 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -795,6 +795,7 @@ func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]* // The returned list of Chart paths is ordered from "leaf to root" so we can issue updates in // the right order when iterating over this list. func (m *Manager) locateLocalDependencies(baseChartPath string, resursive bool) ([]string, error) { + slog.Debug("locating local dependencies", "baseChartPath", baseChartPath, "resursive", resursive) reversedDeps := []string{} baseChart, err := m.loadChartDir(baseChartPath) From 4c28fc52170864be1302a3c4818fc110c8e611d7 Mon Sep 17 00:00:00 2001 From: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> Date: Fri, 19 Sep 2025 17:38:23 +0200 Subject: [PATCH 7/9] Update pkg/downloader/manager.go Co-authored-by: Evans Mungai Signed-off-by: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> --- pkg/downloader/manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index 9c05b5b22..0477e7e62 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -172,6 +172,7 @@ func (m *Manager) Update(recursive bool) error { } func (m *Manager) doUpdate(chartPath string) error { + slog.Debug("update chart dependencies", "chartPath", chartPath) c, err := m.loadChartDir(chartPath) if err != nil { return err From 98cfec128bf78818ef344b6fb34f49bb1ad148c5 Mon Sep 17 00:00:00 2001 From: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> Date: Sat, 20 Sep 2025 20:16:35 +0200 Subject: [PATCH 8/9] add slog Signed-off-by: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> --- pkg/downloader/manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index 7460cb404..0ce0928ab 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -23,6 +23,7 @@ import ( "io" stdfs "io/fs" "log" + "log/slog" "net/url" "os" "path/filepath" From 9bc78168d4ffadda0d7767d1101300649dddda92 Mon Sep 17 00:00:00 2001 From: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> Date: Sun, 21 Sep 2025 22:59:35 +0200 Subject: [PATCH 9/9] fix failing tests Signed-off-by: Alik Khilazhev <7482065+alikhil@users.noreply.github.com> --- pkg/cmd/dependency.go | 2 +- pkg/cmd/dependency_build_test.go | 62 ++++++++++++++++++ pkg/cmd/helpers_test.go | 2 +- .../install-dependency-update-recursive.txt | 1 + .../template-dependency-update-recursive.txt | 6 -- pkg/cmd/testdata/testcharts/root-0.1.0.tgz | Bin 0 -> 1120 bytes pkg/downloader/manager.go | 20 +++--- 7 files changed, 75 insertions(+), 18 deletions(-) create mode 100644 pkg/cmd/testdata/testcharts/root-0.1.0.tgz diff --git a/pkg/cmd/dependency.go b/pkg/cmd/dependency.go index bb7dfe707..b3fc00552 100644 --- a/pkg/cmd/dependency.go +++ b/pkg/cmd/dependency.go @@ -133,5 +133,5 @@ func addDependencySubcommandFlags(f *pflag.FlagSet, client *action.Dependency) { f.BoolVar(&client.InsecureSkipTLSverify, "insecure-skip-tls-verify", false, "skip tls certificate checks for the chart download") f.BoolVar(&client.PlainHTTP, "plain-http", false, "use insecure HTTP connections for the chart download") f.StringVar(&client.CaFile, "ca-file", "", "verify certificates of HTTPS-enabled servers using this CA bundle") - f.BoolVar(&client.BuildOrUpdateRecursive, "recursive", false, "build dependencies recursively") + f.BoolVar(&client.BuildOrUpdateRecursive, "recursive", false, "build or update dependencies recursively") } diff --git a/pkg/cmd/dependency_build_test.go b/pkg/cmd/dependency_build_test.go index a3473301d..9d031457a 100644 --- a/pkg/cmd/dependency_build_test.go +++ b/pkg/cmd/dependency_build_test.go @@ -22,6 +22,7 @@ import ( "strings" "testing" + chart "helm.sh/helm/v4/pkg/chart/v2" chartutil "helm.sh/helm/v4/pkg/chart/v2/util" "helm.sh/helm/v4/pkg/provenance" "helm.sh/helm/v4/pkg/repo/v1" @@ -162,3 +163,64 @@ func TestDependencyBuildCmdWithHelmV2Hash(t *testing.T) { t.Fatal(err) } } + +// createTestingChart creates a basic chart that depends on reqtest-0.1.0 +// +// The baseURL can be used to point to a particular repository server. +func createTestingChartWithRecursion(t *testing.T, dest, name, baseURL string) { + t.Helper() + cfile := &chart.Chart{ + Metadata: &chart.Metadata{ + APIVersion: chart.APIVersionV2, + Name: name, + Version: "1.2.3", + Dependencies: []*chart.Dependency{ + {Name: "reqtest", Version: "0.1.0", Repository: baseURL}, + {Name: "compressedchart", Version: "0.1.0", Repository: baseURL}, + {Name: "root", Version: "0.1.0", Repository: baseURL}, + }, + }, + } + if err := chartutil.SaveDir(cfile, dest); err != nil { + t.Fatal(err) + } +} + +func TestDependencyBuildCmdRecursive(t *testing.T) { + srv := repotest.NewTempServer( + t, + repotest.WithChartSourceGlob("testdata/testcharts/*.tgz"), + ) + defer srv.Stop() + rootDir := srv.Root() + + srv.LinkIndices() + ociSrv, err := repotest.NewOCIServer(t, rootDir) + if err != nil { + t.Fatal(err) + } + + dir := func(p ...string) string { + return filepath.Join(append([]string{rootDir}, p...)...) + } + + ociChartName := "oci-depending-chart" + c := createTestingMetadataForOCI(ociChartName, ociSrv.RegistryURL) + if _, err := chartutil.Save(c, ociSrv.Dir); err != nil { + t.Fatal(err) + } + ociSrv.Run(t, repotest.WithDependingChart(c)) + + chartname := "chart-with-multi-level-deps" + createTestingChartWithRecursion(t, dir(), chartname, srv.URL()) + repoFile := filepath.Join(dir(), "repositories.yaml") + + cmd := fmt.Sprintf("dependency build '%s' --repository-config %s --repository-cache %s --plain-http --recursive", filepath.Join(rootDir, chartname), repoFile, rootDir) + _, out, err := executeActionCommand(cmd) + + // In the first pass, we basically want the same results as an update. + if err != nil { + t.Logf("Output: %s", out) + t.Fatal(err) + } +} diff --git a/pkg/cmd/helpers_test.go b/pkg/cmd/helpers_test.go index 3f04a6a9b..5432ff2de 100644 --- a/pkg/cmd/helpers_test.go +++ b/pkg/cmd/helpers_test.go @@ -31,8 +31,8 @@ import ( "helm.sh/helm/v4/internal/test" "helm.sh/helm/v4/pkg/action" + "helm.sh/helm/v4/pkg/chart/common" "helm.sh/helm/v4/pkg/chart/v2/loader" - chartutil "helm.sh/helm/v4/pkg/chart/v2/util" "helm.sh/helm/v4/pkg/cli" kubefake "helm.sh/helm/v4/pkg/kube/fake" release "helm.sh/helm/v4/pkg/release/v1" diff --git a/pkg/cmd/testdata/output/install-dependency-update-recursive.txt b/pkg/cmd/testdata/output/install-dependency-update-recursive.txt index 9b25fbe36..16ece824e 100644 --- a/pkg/cmd/testdata/output/install-dependency-update-recursive.txt +++ b/pkg/cmd/testdata/output/install-dependency-update-recursive.txt @@ -3,4 +3,5 @@ LAST DEPLOYED: Fri Sep 2 22:04:05 1977 NAMESPACE: default STATUS: deployed REVISION: 1 +DESCRIPTION: Install complete TEST SUITE: None diff --git a/pkg/cmd/testdata/output/template-dependency-update-recursive.txt b/pkg/cmd/testdata/output/template-dependency-update-recursive.txt index 46a53092f..5c451ad75 100644 --- a/pkg/cmd/testdata/output/template-dependency-update-recursive.txt +++ b/pkg/cmd/testdata/output/template-dependency-update-recursive.txt @@ -1,11 +1,5 @@ -Hang tight while we grab the latest from your chart repositories... -...Successfully got an update from the "kube-sigs-external-dns" chart repository -Update Complete. ⎈Happy Helming!⎈ Saving 1 charts Deleting outdated charts -Hang tight while we grab the latest from your chart repositories... -...Successfully got an update from the "kube-sigs-external-dns" chart repository -Update Complete. ⎈Happy Helming!⎈ Saving 1 charts Deleting outdated charts --- diff --git a/pkg/cmd/testdata/testcharts/root-0.1.0.tgz b/pkg/cmd/testdata/testcharts/root-0.1.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..265ce8d89c76894ba304ba13d23a9785febdfc30 GIT binary patch literal 1120 zcmV-m1fTmKiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PLH=ZlgvV#<4fuWm#47p_TSH+8(>H|1&THrk6BHrAk#VO|(61 zt8`$71O)>EoG6MS<=nndpQO)IfhSECXOlXy$$B+E$@;J{FfjAs|Ja|4EUQL4N4%(p zXMBO-XyS3z_0gChHOlT zO}+)<1H939hKJJl|AZ$~S+1-AxX=Ifa?Ae?Fzx>?X!^gBlRV*-EJtCMM)Bc<=k@E$ z8W8vPKVz-_r>?8-|DEtb{fDjNI2AtHX-2`%JhvvY;(}M)w-CBfsXcY8HOr#$|8RHj zV6Vywxte3#?SJe5I4%EU7rWa39nssK3Tzky!!S%^raklax0>oPNB?SHv&X4})9q}Uit_!973NFx<-y?f z`8fr6iyV($yhZ0@QZL)=;^L+G^*5vMemOW!CTaGkYI;wd!PgpDZQL&Pra|NbVI4Bbf6R|-BNlv zQ+3nosmVuA9rT&AD{ngG@8$t-p20U?)LSBV2-{j z1pc_%KotVUEk!_o_GRH~<9fX=1T-3r<`bd${jUk;l})f`67T6cK*j9I{)1b%QO6k`OhfCHkO>bp$Ny|@i5{c zrrwy7Krk_Oz@yZa0h2WF>?raC^`r|9xE}K`q0#O6PrOfvZ`(fghPG=1*748BGUxvl zWI-3te>#-d+W%cJ*Z;RRocQ@2{PEav=wy-b!h7rf8^ zn6>}^!yKaLf1S{p|M5Kk?