From 645393351995b375705c56508b68cb16d3a71b4b Mon Sep 17 00:00:00 2001 From: Terry Howe Date: Fri, 28 Nov 2025 15:38:36 -0700 Subject: [PATCH 1/7] feature: automated OCI end-to-end tests Signed-off-by: Terry Howe --- pkg/cmd/oci_e2e_test.go | 313 ++++++++++++++++++ .../testcharts/unicode-chart-0.1.0.tgz | Bin 0 -> 418 bytes .../testcharts/unicode-chart/Chart.yaml | 4 + .../unicode-chart/templates/NOTES.txt | 3 + .../testcharts/unicode-chart/values.yaml | 2 + 5 files changed, 322 insertions(+) create mode 100644 pkg/cmd/oci_e2e_test.go create mode 100644 pkg/cmd/testdata/testcharts/unicode-chart-0.1.0.tgz create mode 100644 pkg/cmd/testdata/testcharts/unicode-chart/Chart.yaml create mode 100644 pkg/cmd/testdata/testcharts/unicode-chart/templates/NOTES.txt create mode 100644 pkg/cmd/testdata/testcharts/unicode-chart/values.yaml diff --git a/pkg/cmd/oci_e2e_test.go b/pkg/cmd/oci_e2e_test.go new file mode 100644 index 000000000..f8c1dc0a1 --- /dev/null +++ b/pkg/cmd/oci_e2e_test.go @@ -0,0 +1,313 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +// TestOCIRegistryGHCREndToEnd tests push and pull against real GitHub Container Registry (GHCR) +// This test requires HELM_RUN_E2E=true, GHCR_USER and GHCR_TOKEN environment variables to be set +func TestOCIRegistryGHCREndToEnd(t *testing.T) { + if os.Getenv("HELM_RUN_E2E") == "" { + t.Skip("Skipping e2e test: HELM_RUN_E2E environment variable not set") + } + + ghcrUser := os.Getenv("GHCR_USER") + ghcrToken := os.Getenv("GHCR_TOKEN") + + if ghcrUser == "" || ghcrToken == "" { + t.Skip("Skipping GHCR test: GHCR_USER and GHCR_TOKEN environment variables must be set") + } + + // Setup test directories + workDir := t.TempDir() + registryConfigPath := filepath.Join(workDir, "config.json") + contentCache := t.TempDir() + + // GHCR registry configuration + ghcrRegistry := "ghcr.io/terryhowe" + + tests := []struct { + name string + chartPath string + chartName string + chartVersion string + repoPath string + pullArgs string + expectPullFile string + expectPullDir bool + }{ + { + name: "Push and pull basic chart to GHCR", + chartPath: "testdata/testcharts/test-0.1.0.tgz", + chartName: "test", + chartVersion: "0.1.0", + repoPath: "helm-e2e-test", + expectPullFile: "./test-0.1.0.tgz", + expectPullDir: false, + }, + { + name: "Push and pull chart with untar from GHCR", + chartPath: "testdata/testcharts/compressedchart-0.1.0.tgz", + chartName: "compressedchart", + chartVersion: "0.1.0", + repoPath: "helm-e2e-test", + pullArgs: "--untar", + expectPullFile: "./compressedchart", + expectPullDir: true, + }, + { + name: "Push and pull chart with hyphens to GHCR", + chartPath: "testdata/testcharts/compressedchart-with-hyphens-0.1.0.tgz", + chartName: "compressedchart-with-hyphens", + chartVersion: "0.1.0", + repoPath: "helm-e2e-test", + expectPullFile: "./compressedchart-with-hyphens-0.1.0.tgz", + expectPullDir: false, + }, + { + name: "Push and pull chart with unicode description to GHCR", + chartPath: "testdata/testcharts/unicode-chart-0.1.0.tgz", + chartName: "unicode-chart", + chartVersion: "0.1.0", + repoPath: "helm-e2e-test", + expectPullFile: "./unicode-chart-0.1.0.tgz", + expectPullDir: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a fresh pull directory for this test + pullDir := filepath.Join(workDir, tt.name) + if err := os.MkdirAll(pullDir, 0755); err != nil { + t.Fatal(err) + } + + // Construct the push remote (repository path only) + pushRemote := fmt.Sprintf("oci://%s/%s", ghcrRegistry, tt.repoPath) + + // Construct the pull reference (includes chart name and version) + pullRef := fmt.Sprintf("oci://%s/%s/%s:%s", + ghcrRegistry, + tt.repoPath, + tt.chartName, + tt.chartVersion) + + // Push the chart to GHCR + pushCmd := fmt.Sprintf("push %s %s --registry-config %s --username %s --password %s", + tt.chartPath, + pushRemote, + registryConfigPath, + ghcrUser, + ghcrToken) + + t.Logf("Executing push command to GHCR: %s", pushCmd) + _, pushOut, pushErr := executeActionCommand(pushCmd) + + if pushErr != nil { + t.Fatalf("push to GHCR failed: %v\nOutput: %s", pushErr, pushOut) + } + t.Logf("Push to GHCR successful. Output: %s", pushOut) + + // Pull the chart back from GHCR + pullCmd := fmt.Sprintf("pull %s -d '%s' --registry-config %s --content-cache %s --username %s --password %s", + pullRef, + pullDir, + registryConfigPath, + contentCache, + ghcrUser, + ghcrToken) + + if tt.pullArgs != "" { + pullCmd += " " + tt.pullArgs + } + + t.Logf("Executing pull command from GHCR: %s", pullCmd) + _, pullOut, pullErr := executeActionCommand(pullCmd) + + if pullErr != nil { + t.Fatalf("pull from GHCR failed: %v\nOutput: %s", pullErr, pullOut) + } + t.Logf("Pull from GHCR successful. Output: %s", pullOut) + + // Verify the pulled file exists + pulledFilePath := filepath.Join(pullDir, tt.expectPullFile) + fi, err := os.Stat(pulledFilePath) + if err != nil { + t.Errorf("expected file at %s but got error: %s", pulledFilePath, err) + } + + // Verify if it's a directory or file as expected + if fi.IsDir() != tt.expectPullDir { + t.Errorf("expected directory=%t, but got directory=%t", tt.expectPullDir, fi.IsDir()) + } + }) + } +} + +// TestOCIRegistryGHCRAuthFailure tests authentication failures with real GHCR +// This test requires HELM_RUN_E2E=true and GHCR_USER environment variable to be set +func TestOCIRegistryGHCRAuthFailure(t *testing.T) { + if os.Getenv("HELM_RUN_E2E") == "" { + t.Skip("Skipping e2e test: HELM_RUN_E2E environment variable not set") + } + + ghcrUser := os.Getenv("GHCR_USER") + + if ghcrUser == "" { + t.Skip("Skipping GHCR auth failure test: GHCR_USER environment variable must be set") + } + + // Setup test directories + workDir := t.TempDir() + registryConfigPath := filepath.Join(workDir, "config.json") + + // GHCR registry configuration + ghcrRegistry := "ghcr.io/terryhowe" + + t.Run("Fail push with invalid credentials to GHCR", func(t *testing.T) { + pushRemote := fmt.Sprintf("oci://%s/helm-e2e-test", ghcrRegistry) + + // Try to push with invalid credentials + pushCmd := fmt.Sprintf("push testdata/testcharts/test-0.1.0.tgz %s --registry-config %s --username %s --password %s", + pushRemote, + registryConfigPath, + ghcrUser, + "invalid-token-12345") + + t.Logf("Executing push command with invalid credentials to GHCR") + _, _, pushErr := executeActionCommand(pushCmd) + + if pushErr == nil { + t.Fatal("expected push to fail with invalid credentials but it succeeded") + } + t.Logf("Got expected authentication error: %v", pushErr) + }) +} + +// TestOCIRegistryGHCRWithKubernetes tests the full end-to-end flow: +// push to GHCR -> install to Kubernetes -> uninstall from Kubernetes +// This test requires HELM_RUN_E2E=true, GHCR_USER and GHCR_TOKEN environment variables and access to a Kubernetes cluster +func TestOCIRegistryGHCRWithKubernetes(t *testing.T) { + if os.Getenv("HELM_RUN_E2E") == "" { + t.Skip("Skipping e2e test: HELM_RUN_E2E environment variable not set") + } + + ghcrUser := os.Getenv("GHCR_USER") + ghcrToken := os.Getenv("GHCR_TOKEN") + + if ghcrUser == "" || ghcrToken == "" { + t.Skip("Skipping GHCR+K8s test: GHCR_USER and GHCR_TOKEN environment variables must be set") + } + + // Setup test directories + workDir := t.TempDir() + registryConfigPath := filepath.Join(workDir, "config.json") + + // GHCR registry configuration + ghcrRegistry := "ghcr.io/terryhowe" + testNamespace := "helm-e2e-test" + + tests := []struct { + name string + chartPath string + chartName string + chartVersion string + repoPath string + releaseName string + }{ + { + name: "Install basic chart from GHCR to K8s", + chartPath: "testdata/testcharts/test-0.1.0.tgz", + chartName: "test", + chartVersion: "0.1.0", + repoPath: "helm-e2e-test", + releaseName: "test-release", + }, + { + name: "Install chart with unicode from GHCR to K8s", + chartPath: "testdata/testcharts/unicode-chart-0.1.0.tgz", + chartName: "unicode-chart", + chartVersion: "0.1.0", + repoPath: "helm-e2e-test", + releaseName: "unicode-release", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Construct the push remote (repository path only) + pushRemote := fmt.Sprintf("oci://%s/%s", ghcrRegistry, tt.repoPath) + + // Construct the OCI reference for installation + ociRef := fmt.Sprintf("oci://%s/%s/%s", + ghcrRegistry, + tt.repoPath, + tt.chartName) + + // Push the chart to GHCR + pushCmd := fmt.Sprintf("push %s %s --registry-config %s --username %s --password %s", + tt.chartPath, + pushRemote, + registryConfigPath, + ghcrUser, + ghcrToken) + + t.Logf("Pushing chart to GHCR: %s", tt.chartName) + _, pushOut, pushErr := executeActionCommand(pushCmd) + if pushErr != nil { + t.Fatalf("push to GHCR failed: %v\nOutput: %s", pushErr, pushOut) + } + t.Logf("Push successful") + + // Install the chart to Kubernetes + installCmd := fmt.Sprintf("install %s %s --version %s --namespace %s --create-namespace --registry-config %s --username %s --password %s --wait --timeout 2m", + tt.releaseName, + ociRef, + tt.chartVersion, + testNamespace, + registryConfigPath, + ghcrUser, + ghcrToken) + + t.Logf("Installing chart to Kubernetes: %s", tt.releaseName) + _, installOut, installErr := executeActionCommand(installCmd) + if installErr != nil { + t.Fatalf("install to Kubernetes failed: %v\nOutput: %s", installErr, installOut) + } + t.Logf("Install successful. Output: %s", installOut) + + // Verify the release is installed by checking for chart resources in the namespace + // Note: We can't use helm list/status here because executeActionCommand creates separate storage contexts + t.Logf("Verifying installation succeeded (output shows deployed status)") + + // Clean up: Delete the namespace which will remove all resources + t.Logf("Cleaning up namespace: %s", testNamespace) + // Note: We'll delete the namespace at the end of all tests, not per-test + }) + } + + // Cleanup: delete the test namespace + t.Logf("Deleting test namespace: %s", testNamespace) + // Using Go's os/exec would be better but for simplicity in tests we'll rely on the namespace being cleaned up manually + // or by the next test run with --create-namespace +} diff --git a/pkg/cmd/testdata/testcharts/unicode-chart-0.1.0.tgz b/pkg/cmd/testdata/testcharts/unicode-chart-0.1.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..9af1ce2c8dfdfce488a3e1d8d96a448e7b3737bd GIT binary patch literal 418 zcmV;T0bTwdiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PL1AOT$1EfHUV;Ja9IBmnJP_txiH64DLsIHAgR(kh@e%DY*Fy z)TOw&h_2Gzzam8h|AUyeU<1;@(4yQo&2^*m^>s!x5mt>L zBDOrQjEGo`NLp=+_=LC~@!Y0wA!)WN1z!$Wx3+13}8s? z9#c`yGhP0NOlc?^1J1dQVc9Eil)pS@0|xWk`}y?o?cokyXHWB+EBLsczRxagNfQ>p zT4&kART6ZFgZ8e=HY{29e@I2aRNVt$lmFOiR{ZyfPmKTDU=I#hpB6&H;_jiJ<`s2w zmI)rwPFhGE05R_VC)WMfEXf4bOf|a4;o*s+FLVt#HopJ9Tk+rbh-duY24PI)07hw1 ziqECeR0uB5U_1tAu?}`=!eBCSY&(p(S}p`!Mxy93wVD;d#;su2l^P5NgW)@$0RRC1 M|0{4sU;q#R0P4fR@Bjb+ literal 0 HcmV?d00001 diff --git a/pkg/cmd/testdata/testcharts/unicode-chart/Chart.yaml b/pkg/cmd/testdata/testcharts/unicode-chart/Chart.yaml new file mode 100644 index 000000000..9f5690f12 --- /dev/null +++ b/pkg/cmd/testdata/testcharts/unicode-chart/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Test chart with unicode Kröpke 日本語 中文 한글 +name: unicode-chart +version: 0.1.0 diff --git a/pkg/cmd/testdata/testcharts/unicode-chart/templates/NOTES.txt b/pkg/cmd/testdata/testcharts/unicode-chart/templates/NOTES.txt new file mode 100644 index 000000000..75ef6ee88 --- /dev/null +++ b/pkg/cmd/testdata/testcharts/unicode-chart/templates/NOTES.txt @@ -0,0 +1,3 @@ +Thank you for installing {{ .Chart.Name }}. + +This chart includes unicode: Kröpke 日本語 中文 한글 diff --git a/pkg/cmd/testdata/testcharts/unicode-chart/values.yaml b/pkg/cmd/testdata/testcharts/unicode-chart/values.yaml new file mode 100644 index 000000000..f5782aac6 --- /dev/null +++ b/pkg/cmd/testdata/testcharts/unicode-chart/values.yaml @@ -0,0 +1,2 @@ +# Default values for unicode-chart +replicaCount: 1 From 516d83d069c562197776db929065bf5d21e76f5a Mon Sep 17 00:00:00 2001 From: Terry Howe Date: Mon, 21 Sep 2026 06:49:28 -0600 Subject: [PATCH 2/7] test: address review feedback on OCI e2e tests Move the OCI end-to-end tests out of package cmd into a dedicated test/e2e package behind an 'e2e' build tag, and drive a real helm binary rather than executeActionCommand. - Parameterize the registry, namespace, and credentials so the tests run against any OCI registry, not a hard-coded personal GHCR namespace. - Isolate each run's repositories and namespace with a random run ID. - Fail rather than skip when required configuration is missing, since the build tag is already an explicit opt-in. - Pass the registry password over stdin so credentials never reach argv or test output. - Assert an authentication-specific failure in the invalid credential test instead of accepting any error. - Install through a real helm binary against the ambient kubecontext, so the Kubernetes test no longer runs against the fake kube client. - Stop dereferencing the os.Stat result after a stat error. - Add a 'make test-e2e' target and test/e2e/README.md. Signed-off-by: Terry Howe --- Makefile | 9 + pkg/cmd/oci_e2e_test.go | 313 ------------------ test/e2e/README.md | 43 +++ test/e2e/helper_test.go | 246 ++++++++++++++ test/e2e/oci_test.go | 247 ++++++++++++++ .../testcharts/compressedchart-0.1.0.tgz | Bin 0 -> 477 bytes .../compressedchart-with-hyphens-0.1.0.tgz | Bin 0 -> 548 bytes test/e2e/testdata/testcharts/test-0.1.0.tgz | Bin 0 -> 319 bytes .../testcharts/unicode-chart-0.1.0.tgz | Bin 0 -> 418 bytes 9 files changed, 545 insertions(+), 313 deletions(-) delete mode 100644 pkg/cmd/oci_e2e_test.go create mode 100644 test/e2e/README.md create mode 100644 test/e2e/helper_test.go create mode 100644 test/e2e/oci_test.go create mode 100644 test/e2e/testdata/testcharts/compressedchart-0.1.0.tgz create mode 100644 test/e2e/testdata/testcharts/compressedchart-with-hyphens-0.1.0.tgz create mode 100644 test/e2e/testdata/testcharts/test-0.1.0.tgz create mode 100644 test/e2e/testdata/testcharts/unicode-chart-0.1.0.tgz diff --git a/Makefile b/Makefile index d7422bf23..e8be16bd8 100644 --- a/Makefile +++ b/Makefile @@ -127,6 +127,15 @@ test-coverage: @echo "==> Running unit tests with coverage: $(PKG) <==" @ ./scripts/coverage.sh $(PKG) +# End-to-end tests run a real helm binary against a real OCI registry, and +# optionally a real Kubernetes cluster. They are behind the 'e2e' build tag and +# require configuration; see test/e2e/README.md. +.PHONY: test-e2e +test-e2e: + @echo + @echo "==> Running end-to-end tests <==" + go test $(GOFLAGS) -tags e2e -count=1 -v ./test/e2e/... + .PHONY: test-style test-style: golangci-lint run ./... diff --git a/pkg/cmd/oci_e2e_test.go b/pkg/cmd/oci_e2e_test.go deleted file mode 100644 index f8c1dc0a1..000000000 --- a/pkg/cmd/oci_e2e_test.go +++ /dev/null @@ -1,313 +0,0 @@ -/* -Copyright The Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "fmt" - "os" - "path/filepath" - "testing" -) - -// TestOCIRegistryGHCREndToEnd tests push and pull against real GitHub Container Registry (GHCR) -// This test requires HELM_RUN_E2E=true, GHCR_USER and GHCR_TOKEN environment variables to be set -func TestOCIRegistryGHCREndToEnd(t *testing.T) { - if os.Getenv("HELM_RUN_E2E") == "" { - t.Skip("Skipping e2e test: HELM_RUN_E2E environment variable not set") - } - - ghcrUser := os.Getenv("GHCR_USER") - ghcrToken := os.Getenv("GHCR_TOKEN") - - if ghcrUser == "" || ghcrToken == "" { - t.Skip("Skipping GHCR test: GHCR_USER and GHCR_TOKEN environment variables must be set") - } - - // Setup test directories - workDir := t.TempDir() - registryConfigPath := filepath.Join(workDir, "config.json") - contentCache := t.TempDir() - - // GHCR registry configuration - ghcrRegistry := "ghcr.io/terryhowe" - - tests := []struct { - name string - chartPath string - chartName string - chartVersion string - repoPath string - pullArgs string - expectPullFile string - expectPullDir bool - }{ - { - name: "Push and pull basic chart to GHCR", - chartPath: "testdata/testcharts/test-0.1.0.tgz", - chartName: "test", - chartVersion: "0.1.0", - repoPath: "helm-e2e-test", - expectPullFile: "./test-0.1.0.tgz", - expectPullDir: false, - }, - { - name: "Push and pull chart with untar from GHCR", - chartPath: "testdata/testcharts/compressedchart-0.1.0.tgz", - chartName: "compressedchart", - chartVersion: "0.1.0", - repoPath: "helm-e2e-test", - pullArgs: "--untar", - expectPullFile: "./compressedchart", - expectPullDir: true, - }, - { - name: "Push and pull chart with hyphens to GHCR", - chartPath: "testdata/testcharts/compressedchart-with-hyphens-0.1.0.tgz", - chartName: "compressedchart-with-hyphens", - chartVersion: "0.1.0", - repoPath: "helm-e2e-test", - expectPullFile: "./compressedchart-with-hyphens-0.1.0.tgz", - expectPullDir: false, - }, - { - name: "Push and pull chart with unicode description to GHCR", - chartPath: "testdata/testcharts/unicode-chart-0.1.0.tgz", - chartName: "unicode-chart", - chartVersion: "0.1.0", - repoPath: "helm-e2e-test", - expectPullFile: "./unicode-chart-0.1.0.tgz", - expectPullDir: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Create a fresh pull directory for this test - pullDir := filepath.Join(workDir, tt.name) - if err := os.MkdirAll(pullDir, 0755); err != nil { - t.Fatal(err) - } - - // Construct the push remote (repository path only) - pushRemote := fmt.Sprintf("oci://%s/%s", ghcrRegistry, tt.repoPath) - - // Construct the pull reference (includes chart name and version) - pullRef := fmt.Sprintf("oci://%s/%s/%s:%s", - ghcrRegistry, - tt.repoPath, - tt.chartName, - tt.chartVersion) - - // Push the chart to GHCR - pushCmd := fmt.Sprintf("push %s %s --registry-config %s --username %s --password %s", - tt.chartPath, - pushRemote, - registryConfigPath, - ghcrUser, - ghcrToken) - - t.Logf("Executing push command to GHCR: %s", pushCmd) - _, pushOut, pushErr := executeActionCommand(pushCmd) - - if pushErr != nil { - t.Fatalf("push to GHCR failed: %v\nOutput: %s", pushErr, pushOut) - } - t.Logf("Push to GHCR successful. Output: %s", pushOut) - - // Pull the chart back from GHCR - pullCmd := fmt.Sprintf("pull %s -d '%s' --registry-config %s --content-cache %s --username %s --password %s", - pullRef, - pullDir, - registryConfigPath, - contentCache, - ghcrUser, - ghcrToken) - - if tt.pullArgs != "" { - pullCmd += " " + tt.pullArgs - } - - t.Logf("Executing pull command from GHCR: %s", pullCmd) - _, pullOut, pullErr := executeActionCommand(pullCmd) - - if pullErr != nil { - t.Fatalf("pull from GHCR failed: %v\nOutput: %s", pullErr, pullOut) - } - t.Logf("Pull from GHCR successful. Output: %s", pullOut) - - // Verify the pulled file exists - pulledFilePath := filepath.Join(pullDir, tt.expectPullFile) - fi, err := os.Stat(pulledFilePath) - if err != nil { - t.Errorf("expected file at %s but got error: %s", pulledFilePath, err) - } - - // Verify if it's a directory or file as expected - if fi.IsDir() != tt.expectPullDir { - t.Errorf("expected directory=%t, but got directory=%t", tt.expectPullDir, fi.IsDir()) - } - }) - } -} - -// TestOCIRegistryGHCRAuthFailure tests authentication failures with real GHCR -// This test requires HELM_RUN_E2E=true and GHCR_USER environment variable to be set -func TestOCIRegistryGHCRAuthFailure(t *testing.T) { - if os.Getenv("HELM_RUN_E2E") == "" { - t.Skip("Skipping e2e test: HELM_RUN_E2E environment variable not set") - } - - ghcrUser := os.Getenv("GHCR_USER") - - if ghcrUser == "" { - t.Skip("Skipping GHCR auth failure test: GHCR_USER environment variable must be set") - } - - // Setup test directories - workDir := t.TempDir() - registryConfigPath := filepath.Join(workDir, "config.json") - - // GHCR registry configuration - ghcrRegistry := "ghcr.io/terryhowe" - - t.Run("Fail push with invalid credentials to GHCR", func(t *testing.T) { - pushRemote := fmt.Sprintf("oci://%s/helm-e2e-test", ghcrRegistry) - - // Try to push with invalid credentials - pushCmd := fmt.Sprintf("push testdata/testcharts/test-0.1.0.tgz %s --registry-config %s --username %s --password %s", - pushRemote, - registryConfigPath, - ghcrUser, - "invalid-token-12345") - - t.Logf("Executing push command with invalid credentials to GHCR") - _, _, pushErr := executeActionCommand(pushCmd) - - if pushErr == nil { - t.Fatal("expected push to fail with invalid credentials but it succeeded") - } - t.Logf("Got expected authentication error: %v", pushErr) - }) -} - -// TestOCIRegistryGHCRWithKubernetes tests the full end-to-end flow: -// push to GHCR -> install to Kubernetes -> uninstall from Kubernetes -// This test requires HELM_RUN_E2E=true, GHCR_USER and GHCR_TOKEN environment variables and access to a Kubernetes cluster -func TestOCIRegistryGHCRWithKubernetes(t *testing.T) { - if os.Getenv("HELM_RUN_E2E") == "" { - t.Skip("Skipping e2e test: HELM_RUN_E2E environment variable not set") - } - - ghcrUser := os.Getenv("GHCR_USER") - ghcrToken := os.Getenv("GHCR_TOKEN") - - if ghcrUser == "" || ghcrToken == "" { - t.Skip("Skipping GHCR+K8s test: GHCR_USER and GHCR_TOKEN environment variables must be set") - } - - // Setup test directories - workDir := t.TempDir() - registryConfigPath := filepath.Join(workDir, "config.json") - - // GHCR registry configuration - ghcrRegistry := "ghcr.io/terryhowe" - testNamespace := "helm-e2e-test" - - tests := []struct { - name string - chartPath string - chartName string - chartVersion string - repoPath string - releaseName string - }{ - { - name: "Install basic chart from GHCR to K8s", - chartPath: "testdata/testcharts/test-0.1.0.tgz", - chartName: "test", - chartVersion: "0.1.0", - repoPath: "helm-e2e-test", - releaseName: "test-release", - }, - { - name: "Install chart with unicode from GHCR to K8s", - chartPath: "testdata/testcharts/unicode-chart-0.1.0.tgz", - chartName: "unicode-chart", - chartVersion: "0.1.0", - repoPath: "helm-e2e-test", - releaseName: "unicode-release", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Construct the push remote (repository path only) - pushRemote := fmt.Sprintf("oci://%s/%s", ghcrRegistry, tt.repoPath) - - // Construct the OCI reference for installation - ociRef := fmt.Sprintf("oci://%s/%s/%s", - ghcrRegistry, - tt.repoPath, - tt.chartName) - - // Push the chart to GHCR - pushCmd := fmt.Sprintf("push %s %s --registry-config %s --username %s --password %s", - tt.chartPath, - pushRemote, - registryConfigPath, - ghcrUser, - ghcrToken) - - t.Logf("Pushing chart to GHCR: %s", tt.chartName) - _, pushOut, pushErr := executeActionCommand(pushCmd) - if pushErr != nil { - t.Fatalf("push to GHCR failed: %v\nOutput: %s", pushErr, pushOut) - } - t.Logf("Push successful") - - // Install the chart to Kubernetes - installCmd := fmt.Sprintf("install %s %s --version %s --namespace %s --create-namespace --registry-config %s --username %s --password %s --wait --timeout 2m", - tt.releaseName, - ociRef, - tt.chartVersion, - testNamespace, - registryConfigPath, - ghcrUser, - ghcrToken) - - t.Logf("Installing chart to Kubernetes: %s", tt.releaseName) - _, installOut, installErr := executeActionCommand(installCmd) - if installErr != nil { - t.Fatalf("install to Kubernetes failed: %v\nOutput: %s", installErr, installOut) - } - t.Logf("Install successful. Output: %s", installOut) - - // Verify the release is installed by checking for chart resources in the namespace - // Note: We can't use helm list/status here because executeActionCommand creates separate storage contexts - t.Logf("Verifying installation succeeded (output shows deployed status)") - - // Clean up: Delete the namespace which will remove all resources - t.Logf("Cleaning up namespace: %s", testNamespace) - // Note: We'll delete the namespace at the end of all tests, not per-test - }) - } - - // Cleanup: delete the test namespace - t.Logf("Deleting test namespace: %s", testNamespace) - // Using Go's os/exec would be better but for simplicity in tests we'll rely on the namespace being cleaned up manually - // or by the next test run with --create-namespace -} diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 000000000..799fb00d4 --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,43 @@ +# Helm end-to-end tests + +These tests exercise a real `helm` binary against real external systems: an OCI +registry, and optionally a Kubernetes cluster. They are guarded by the `e2e` +build tag, so they are never compiled or run by `make test`. + +Unlike the unit tests, nothing here is faked. The registry is a real registry, +and the Kubernetes test installs into whatever cluster your current kubecontext +points at. + +## Running + +```console +$ export HELM_E2E_REGISTRY=ghcr.io/your-org +$ export HELM_E2E_USERNAME=your-user +$ export HELM_E2E_PASSWORD=your-token +$ make test-e2e +``` + +Any OCI registry works, not just GHCR. To run against a local registry: + +```console +$ export HELM_E2E_REGISTRY=localhost:5000/charts +$ export HELM_E2E_PLAIN_HTTP=true +``` + +## Configuration + +| Variable | Required | Description | +| ---------------------- | -------- | ---------------------------------------------------------------------------------------------- | +| `HELM_E2E_REGISTRY` | yes | Registry namespace to push to, without a scheme (e.g. `ghcr.io/your-org`). | +| `HELM_E2E_USERNAME` | yes | Username for the registry. | +| `HELM_E2E_PASSWORD` | yes | Password or token for the registry. Passed to helm on stdin and never logged. | +| `HELM_E2E_BIN` | no | Path to a prebuilt helm binary. Defaults to building one from the working tree. | +| `HELM_E2E_PLAIN_HTTP` | no | Set to use plain HTTP, for registries without TLS. | +| `HELM_E2E_KUBERNETES` | no | Set to run the install test against the current kubecontext. Off by default because it mutates. | +| `HELM_E2E_NAMESPACE` | no | Namespace for the Kubernetes test. Defaults to a unique `helm-e2e-` namespace. | + +Because the `e2e` build tag is already an explicit opt-in, a missing required +variable fails the test rather than silently skipping it. + +Each run pushes to repositories suffixed with a random run ID, so concurrent +runs and repeated runs in a shared namespace do not collide. diff --git a/test/e2e/helper_test.go b/test/e2e/helper_test.go new file mode 100644 index 000000000..91ed186ac --- /dev/null +++ b/test/e2e/helper_test.go @@ -0,0 +1,246 @@ +//go:build e2e + +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package e2e contains end-to-end tests that exercise a real helm binary +// against real external systems (an OCI registry, and optionally a Kubernetes +// cluster). +// +// These tests are excluded from the normal build by the "e2e" build tag and +// are never run by `make test`. Run them with `make test-e2e` after exporting +// the configuration described in the package README. +package e2e + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Environment variables used to configure the end-to-end tests. +const ( + // envRegistry is the registry namespace charts are pushed to, without a + // scheme, e.g. "ghcr.io/example-org" or "localhost:5000/charts". + envRegistry = "HELM_E2E_REGISTRY" + // envUsername is the username used to authenticate to envRegistry. + envUsername = "HELM_E2E_USERNAME" + // envPassword is the password or token used to authenticate to envRegistry. + envPassword = "HELM_E2E_PASSWORD" + // envHelmBin points at a prebuilt helm binary. When unset, one is built + // from the working tree. + envHelmBin = "HELM_E2E_BIN" + // envPlainHTTP requests plain HTTP for registries without TLS, which is + // useful when testing against a local registry. + envPlainHTTP = "HELM_E2E_PLAIN_HTTP" + // envKubernetes opts in to the tests that install charts into a real + // Kubernetes cluster using the ambient kubeconfig. + envKubernetes = "HELM_E2E_KUBERNETES" + // envNamespace is the namespace the Kubernetes tests install into. + envNamespace = "HELM_E2E_NAMESPACE" +) + +// harness carries the resolved configuration shared by the end-to-end tests. +type harness struct { + // helmBin is an absolute path to the helm binary under test. + helmBin string + // registry is the registry namespace charts are pushed to. + registry string + // username and password authenticate to the registry. They are only ever + // passed to helm over stdin and are never logged. + username string + password string + // plainHTTP is true when the registry should be reached over HTTP. + plainHTTP bool + // configPath is the registry credential file used for this run. + configPath string + // runID isolates the repositories written by a single test run so that + // concurrent or repeated runs do not collide. + runID string +} + +// newHarness resolves the end-to-end configuration, failing the test when a +// required value is missing. Because the "e2e" build tag is an explicit opt-in, +// a missing setting is a configuration error rather than a reason to skip. +func newHarness(t *testing.T) *harness { + t.Helper() + + h := &harness{ + registry: requireEnv(t, envRegistry), + username: requireEnv(t, envUsername), + password: requireEnv(t, envPassword), + plainHTTP: os.Getenv(envPlainHTTP) != "", + helmBin: helmBinary(t), + runID: runID(t), + } + return h +} + +// requireEnv returns the value of the named environment variable, failing the +// test with an actionable message when it is unset. +func requireEnv(t *testing.T, name string) string { + t.Helper() + v := os.Getenv(name) + if v == "" { + t.Fatalf("%s must be set to run the end-to-end tests; see test/e2e/README.md", name) + } + return v +} + +// helmBinary returns the helm binary to exercise, building one from the +// working tree when HELM_E2E_BIN is not set. +func helmBinary(t *testing.T) string { + t.Helper() + + if bin := os.Getenv(envHelmBin); bin != "" { + abs, err := filepath.Abs(bin) + if err != nil { + t.Fatalf("resolving %s=%q: %v", envHelmBin, bin, err) + } + if _, err := os.Stat(abs); err != nil { + t.Fatalf("%s=%q is not usable: %v", envHelmBin, bin, err) + } + return abs + } + + bin := filepath.Join(t.TempDir(), "helm") + build := exec.Command("go", "build", "-o", bin, "helm.sh/helm/v4/cmd/helm") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("building helm: %v\n%s", err, out) + } + return bin +} + +// runID returns a short random identifier unique to this test run. +func runID(t *testing.T) string { + t.Helper() + b := make([]byte, 4) + if _, err := rand.Read(b); err != nil { + t.Fatalf("generating run id: %v", err) + } + return hex.EncodeToString(b) +} + +// registryHost returns the host portion of the configured registry, which is +// what `helm registry login` expects. +func (h *harness) registryHost() string { + host, _, _ := strings.Cut(h.registry, "/") + return host +} + +// repo returns an isolated repository path for this run, so that a shared +// registry namespace can serve many runs without interference. +func (h *harness) repo(name string) string { + return fmt.Sprintf("%s/%s-%s", h.registry, name, h.runID) +} + +// ref returns a full OCI reference for a chart within an isolated repository. +func (h *harness) ref(repo, chart, version string) string { + if version == "" { + return fmt.Sprintf("oci://%s/%s", repo, chart) + } + return fmt.Sprintf("oci://%s/%s:%s", repo, chart, version) +} + +// plainHTTPCommands are the helm subcommands that accept --plain-http. The +// flag is only appended for these so that the harness can still run commands +// such as `status` and `uninstall` against a plain HTTP registry. +var plainHTTPCommands = map[string]bool{"push": true, "pull": true, "install": true, "upgrade": true} + +// helm runs the helm binary with the given arguments and returns its combined +// output. Arguments are logged, so callers must never pass a credential. +func (h *harness) helm(t *testing.T, args ...string) (string, error) { + t.Helper() + if h.plainHTTP && len(args) > 0 && plainHTTPCommands[args[0]] { + args = append(args, "--plain-http") + } + t.Logf("helm %s", strings.Join(args, " ")) + cmd := exec.Command(h.helmBin, args...) + cmd.Env = append(os.Environ(), "HELM_REGISTRY_CONFIG="+h.registryConfig(t)) + out, err := cmd.CombinedOutput() + return string(out), err +} + +// mustHelm runs helm and fails the test if the command does not succeed. +func (h *harness) mustHelm(t *testing.T, args ...string) string { + t.Helper() + out, err := h.helm(t, args...) + if err != nil { + t.Fatalf("helm %s failed: %v\n%s", strings.Join(args, " "), err, out) + } + return out +} + +// registryConfig returns the path to this run's registry credential file. The +// file lives under a per-test temporary directory so credentials never leak +// into the developer's real helm configuration. +func (h *harness) registryConfig(t *testing.T) string { + t.Helper() + if h.configPath == "" { + h.configPath = filepath.Join(t.TempDir(), "registry-config.json") + } + return h.configPath +} + +// login authenticates to the configured registry. The password is written to +// helm's stdin rather than passed as a flag so it cannot appear in process +// listings or test output. +func (h *harness) login(t *testing.T, password string) (string, error) { + t.Helper() + args := []string{"registry", "login", h.registryHost(), "--username", h.username, "--password-stdin"} + if h.plainHTTP { + args = append(args, "--plain-http") + } + t.Logf("helm %s", strings.Join(args, " ")) + cmd := exec.Command(h.helmBin, args...) + cmd.Env = append(os.Environ(), "HELM_REGISTRY_CONFIG="+h.registryConfig(t)) + cmd.Stdin = strings.NewReader(password) + out, err := cmd.CombinedOutput() + return string(out), err +} + +// mustLogin authenticates with the configured credential and arranges for a +// logout once the test completes. +func (h *harness) mustLogin(t *testing.T) { + t.Helper() + out, err := h.login(t, h.password) + if err != nil { + t.Fatalf("registry login to %s failed: %v\n%s", h.registryHost(), err, out) + } + t.Cleanup(func() { + if out, err := h.helm(t, "registry", "logout", h.registryHost()); err != nil { + t.Logf("registry logout failed (ignored): %v\n%s", err, out) + } + }) +} + +// chart returns the path to a chart archive in this package's testdata. +func chart(t *testing.T, name string) string { + t.Helper() + path, err := filepath.Abs(filepath.Join("testdata", "testcharts", name)) + if err != nil { + t.Fatalf("resolving chart %q: %v", name, err) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("chart %q is missing: %v", name, err) + } + return path +} diff --git a/test/e2e/oci_test.go b/test/e2e/oci_test.go new file mode 100644 index 000000000..03a382d84 --- /dev/null +++ b/test/e2e/oci_test.go @@ -0,0 +1,247 @@ +//go:build e2e + +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestOCIRegistryPushPull pushes chart archives to the configured OCI registry +// and pulls them back, verifying that the artifact round-trips intact. +func TestOCIRegistryPushPull(t *testing.T) { + h := newHarness(t) + h.mustLogin(t) + + repo := h.repo("push-pull") + contentCache := t.TempDir() + + tests := []struct { + name string + chart string + chartName string + chartVersion string + pullArgs []string + expectPullFile string + expectPullDir bool + }{ + { + name: "basic chart", + chart: "test-0.1.0.tgz", + chartName: "test", + chartVersion: "0.1.0", + expectPullFile: "test-0.1.0.tgz", + }, + { + name: "chart pulled with untar", + chart: "compressedchart-0.1.0.tgz", + chartName: "compressedchart", + chartVersion: "0.1.0", + pullArgs: []string{"--untar"}, + expectPullFile: "compressedchart", + expectPullDir: true, + }, + { + name: "chart name with hyphens", + chart: "compressedchart-with-hyphens-0.1.0.tgz", + chartName: "compressedchart-with-hyphens", + chartVersion: "0.1.0", + expectPullFile: "compressedchart-with-hyphens-0.1.0.tgz", + }, + { + name: "chart with unicode description", + chart: "unicode-chart-0.1.0.tgz", + chartName: "unicode-chart", + chartVersion: "0.1.0", + expectPullFile: "unicode-chart-0.1.0.tgz", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pullDir := t.TempDir() + + h.mustHelm(t, "push", chart(t, tt.chart), "oci://"+repo) + + pullArgs := append([]string{ + "pull", h.ref(repo, tt.chartName, tt.chartVersion), + "--destination", pullDir, + "--content-cache", contentCache, + }, tt.pullArgs...) + h.mustHelm(t, pullArgs...) + + pulled := filepath.Join(pullDir, tt.expectPullFile) + fi, err := os.Stat(pulled) + if err != nil { + t.Fatalf("expected pulled chart at %s: %v", pulled, err) + } + if fi.IsDir() != tt.expectPullDir { + t.Errorf("expected directory=%t, got directory=%t", tt.expectPullDir, fi.IsDir()) + } + }) + } +} + +// TestOCIRegistryInvalidCredentials verifies that the registry rejects a bad +// credential with an authentication error, rather than any error at all. +func TestOCIRegistryInvalidCredentials(t *testing.T) { + h := newHarness(t) + + // Log in with a deliberately wrong password against the same registry and + // namespace the successful tests use, so the only variable is the + // credential itself. + out, err := h.login(t, "invalid-"+h.runID) + if err == nil { + t.Fatal("expected registry login to fail with an invalid credential, but it succeeded") + } + if !isAuthError(out) { + t.Fatalf("expected an authentication failure, got a different error:\n%s", out) + } + + // Seed the credential store directly with the same wrong password, so the + // push reaches the registry and is rejected by it rather than failing + // locally for want of any credential at all. + writeRegistryCredential(t, h.registryConfig(t), h.registryHost(), h.username, "invalid-"+h.runID) + + out, err = h.helm(t, "push", chart(t, "test-0.1.0.tgz"), "oci://"+h.repo("auth-failure")) + if err == nil { + t.Fatal("expected push to fail with an invalid credential, but it succeeded") + } + if !isAuthError(out) { + t.Fatalf("expected an authentication failure, got a different error:\n%s", out) + } +} + +// writeRegistryCredential writes a docker-style credential file so a test can +// present a specific credential to the registry without going through +// `helm registry login`, which refuses to store one the registry rejects. +func writeRegistryCredential(t *testing.T, path, host, username, password string) { + t.Helper() + cfg := struct { + Auths map[string]struct { + Auth string `json:"auth"` + } `json:"auths"` + }{ + Auths: map[string]struct { + Auth string `json:"auth"` + }{ + host: {Auth: base64.StdEncoding.EncodeToString([]byte(username + ":" + password))}, + }, + } + b, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("encoding registry credential: %v", err) + } + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatalf("writing registry credential to %s: %v", path, err) + } +} + +// isAuthError reports whether the output describes an authentication or +// authorization failure, as opposed to a network, DNS, or naming error that +// would otherwise make the test pass for the wrong reason. +func isAuthError(out string) bool { + out = strings.ToLower(out) + for _, marker := range []string{ + "401", + "403", + "unauthorized", + "denied", + "authentication required", + "invalid username/password", + } { + if strings.Contains(out, marker) { + return true + } + } + return false +} + +// TestOCIRegistryInstallToKubernetes exercises the full flow of pushing a +// chart to an OCI registry and installing it into a real Kubernetes cluster +// using the ambient kubeconfig. It is opt-in because, unlike the registry +// tests, it mutates a cluster. +func TestOCIRegistryInstallToKubernetes(t *testing.T) { + if os.Getenv(envKubernetes) == "" { + t.Skipf("Skipping Kubernetes end-to-end test: set %s to run it against the current kubecontext", envKubernetes) + } + + h := newHarness(t) + h.mustLogin(t) + + namespace := os.Getenv(envNamespace) + if namespace == "" { + namespace = "helm-e2e-" + h.runID + } + repo := h.repo("install") + + tests := []struct { + name string + chart string + chartName string + chartVersion string + releaseName string + }{ + { + name: "basic chart", + chart: "test-0.1.0.tgz", + chartName: "test", + chartVersion: "0.1.0", + releaseName: "test-release", + }, + { + name: "chart with unicode description", + chart: "unicode-chart-0.1.0.tgz", + chartName: "unicode-chart", + chartVersion: "0.1.0", + releaseName: "unicode-release", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h.mustHelm(t, "push", chart(t, tt.chart), "oci://"+repo) + + t.Cleanup(func() { + if out, err := h.helm(t, "uninstall", tt.releaseName, "--namespace", namespace, "--ignore-not-found", "--wait"); err != nil { + t.Errorf("uninstalling %s failed: %v\n%s", tt.releaseName, err, out) + } + }) + + h.mustHelm(t, "install", tt.releaseName, h.ref(repo, tt.chartName, ""), + "--version", tt.chartVersion, + "--namespace", namespace, + "--create-namespace", + "--wait", + "--timeout", "2m", + ) + + // Read the release back from cluster storage to confirm the + // install really reached the API server. + out := h.mustHelm(t, "status", tt.releaseName, "--namespace", namespace, "--output", "json") + if !strings.Contains(out, `"status":"deployed"`) { + t.Errorf("expected release %s to be deployed, got:\n%s", tt.releaseName, out) + } + }) + } +} diff --git a/test/e2e/testdata/testcharts/compressedchart-0.1.0.tgz b/test/e2e/testdata/testcharts/compressedchart-0.1.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..3c9c24d76063d6a904405b2d6a7a84cb087f1ce4 GIT binary patch literal 477 zcmV<30V4h%iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PL4vi_<_5!26s}F{UjHhD>x0Xejw~|-egB1QU=&JEdrHEIt>CEtv%dd}n=<=92zSKn;o(8OM@#S> zYFc5(0#_R!xxRXQ%zfa$rtiOMiLGgzk94**j`?3M7Jt0|r`i8O7{f;tq39Bbhr@%1 zO-l}zo#EQJ1_D-Jv7w}jF??!Gg4BiJqa;WzF+;Dc zVQyr3R8em|NM&qo0PL4fi`y^|#dG$jc->rDXjD0A64=|)92WW)wwIoYVoz*QSrU?* zG;H^~7dcDXrjTqgP3fZFMMBsbi zPWu0B_M89nr2n%p#E0nPPIpGV%QZGNX)If*N~tSYQG5|qH0t|nfN!leE_nEwltQJ< z5{(E&Ep_!Aj+6*;9W6i9KdlR0WlY0RR8Xa#gbc6aWCh@%&~0 literal 0 HcmV?d00001 diff --git a/test/e2e/testdata/testcharts/test-0.1.0.tgz b/test/e2e/testdata/testcharts/test-0.1.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..9ed772a7f924ffc4e280de8faa67ff88d857f9ca GIT binary patch literal 319 zcmV-F0l@wriwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PK~)O2jY_hI{T)1iW`A+a?QpRj+z*@ji5?8c5TUOpCs~Y($V; zdeC-J@?D0xbY|fH8m7Rz+gJ2ly<=~SH2?t6O%p2sq!nb{6jDj3Wv!Ju6d?CfHHCmx zwn>-*qc7mcv(n@K2soN&^1%%5C}vrnDYV{<9QzXm7VRY)q8q%J9HSTQ;5uts7MJ80 z_fk6S=@`3m{`wMBvV!v3w`G=z{?BM93akA9mi&i$pZedxA^X1tH{y;uoGgpa5Jc0X z%-7y5vjx$Arhc@@6m}1fr~9Xh-}pCYEc(CKiT|}$I{Uu`-`;rR0;9EbuCn`?OeXX1 R`3L|2|NmE1eCz-a007`FmKp#6 literal 0 HcmV?d00001 diff --git a/test/e2e/testdata/testcharts/unicode-chart-0.1.0.tgz b/test/e2e/testdata/testcharts/unicode-chart-0.1.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..9af1ce2c8dfdfce488a3e1d8d96a448e7b3737bd GIT binary patch literal 418 zcmV;T0bTwdiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PL1AOT$1EfHUV;Ja9IBmnJP_txiH64DLsIHAgR(kh@e%DY*Fy z)TOw&h_2Gzzam8h|AUyeU<1;@(4yQo&2^*m^>s!x5mt>L zBDOrQjEGo`NLp=+_=LC~@!Y0wA!)WN1z!$Wx3+13}8s? z9#c`yGhP0NOlc?^1J1dQVc9Eil)pS@0|xWk`}y?o?cokyXHWB+EBLsczRxagNfQ>p zT4&kART6ZFgZ8e=HY{29e@I2aRNVt$lmFOiR{ZyfPmKTDU=I#hpB6&H;_jiJ<`s2w zmI)rwPFhGE05R_VC)WMfEXf4bOf|a4;o*s+FLVt#HopJ9Tk+rbh-duY24PI)07hw1 ziqECeR0uB5U_1tAu?}`=!eBCSY&(p(S}p`!Mxy93wVD;d#;su2l^P5NgW)@$0RRC1 M|0{4sU;q#R0P4fR@Bjb+ literal 0 HcmV?d00001 From 6c5c300def1846c7e9402d0c38a0488eb463a41a Mon Sep 17 00:00:00 2001 From: Terry Howe Date: Mon, 21 Sep 2026 07:38:48 -0600 Subject: [PATCH 3/7] test: address second round of review feedback on OCI e2e tests - Verify pulled chart contents against the source fixture instead of only checking that a path of the expected name exists, so a registry serving a different artifact under the same tag fails the test. - Skip the invalid-credential test against a registry that serves anonymous access, where there is no authentication to exercise. - Delete the namespace the Kubernetes test creates, and wait for it to go, so runs do not accumulate namespaces. A caller-supplied HELM_E2E_NAMESPACE is left in place. Signed-off-by: Terry Howe --- test/e2e/README.md | 9 +++ test/e2e/oci_test.go | 161 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 169 insertions(+), 1 deletion(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 799fb00d4..678cec038 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -24,6 +24,11 @@ $ export HELM_E2E_REGISTRY=localhost:5000/charts $ export HELM_E2E_PLAIN_HTTP=true ``` +A default `registry:2` deployment serves anonymous access and accepts any +credential. Against such a registry there is no authentication to exercise, so +the invalid-credential test detects this and skips itself. Run against an +auth-enabled registry to cover that path. + ## Configuration | Variable | Required | Description | @@ -41,3 +46,7 @@ variable fails the test rather than silently skipping it. Each run pushes to repositories suffixed with a random run ID, so concurrent runs and repeated runs in a shared namespace do not collide. + +The Kubernetes test deletes the namespace it creates. If you supply +`HELM_E2E_NAMESPACE`, that namespace is left in place and only the releases are +uninstalled. diff --git a/test/e2e/oci_test.go b/test/e2e/oci_test.go index 03a382d84..92699fcbe 100644 --- a/test/e2e/oci_test.go +++ b/test/e2e/oci_test.go @@ -19,12 +19,24 @@ limitations under the License. package e2e import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" "encoding/base64" "encoding/json" + "io" "os" "path/filepath" "strings" "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" ) // TestOCIRegistryPushPull pushes chart archives to the configured OCI registry @@ -98,6 +110,10 @@ func TestOCIRegistryPushPull(t *testing.T) { if fi.IsDir() != tt.expectPullDir { t.Errorf("expected directory=%t, got directory=%t", tt.expectPullDir, fi.IsDir()) } + + // Verify the round-trip actually returned the chart we pushed, + // not merely some artifact under the same tag. + assertChartMatches(t, chart(t, tt.chart), pullDir, tt.expectPullDir) }) } } @@ -112,7 +128,10 @@ func TestOCIRegistryInvalidCredentials(t *testing.T) { // credential itself. out, err := h.login(t, "invalid-"+h.runID) if err == nil { - t.Fatal("expected registry login to fail with an invalid credential, but it succeeded") + // An anonymous registry, such as a default registry:2 deployment, + // accepts any credential. There is no authentication to exercise, so + // skip rather than report a failure the registry cannot produce. + t.Skipf("Skipping invalid credential test: %s accepts unauthenticated access", h.registryHost()) } if !isAuthError(out) { t.Fatalf("expected an authentication failure, got a different error:\n%s", out) @@ -189,9 +208,13 @@ func TestOCIRegistryInstallToKubernetes(t *testing.T) { h := newHarness(t) h.mustLogin(t) + // A caller-supplied namespace is left alone; one we create for the run is + // removed again so repeated runs do not accumulate namespaces in the + // cluster. namespace := os.Getenv(envNamespace) if namespace == "" { namespace = "helm-e2e-" + h.runID + t.Cleanup(func() { deleteNamespace(t, namespace) }) } repo := h.repo("install") @@ -245,3 +268,139 @@ func TestOCIRegistryInstallToKubernetes(t *testing.T) { }) } } + +// assertChartMatches verifies that what was pulled into pullDir is the same +// chart as the fixture at src. Existence and file-vs-directory checks alone +// would pass if the registry served a different artifact under the same tag. +// +// A chart pulled without --untar is the pushed archive verbatim, so it is +// compared byte for byte. With --untar, helm expands the archive, so every +// regular file in the source archive is compared against its extracted +// counterpart. +func assertChartMatches(t *testing.T, src, pullDir string, untarred bool) { + t.Helper() + + if !untarred { + assertFilesEqual(t, src, filepath.Join(pullDir, filepath.Base(src))) + return + } + + want := archiveFiles(t, src) + if len(want) == 0 { + t.Fatalf("fixture %s contains no files", src) + } + for name, content := range want { + extracted := filepath.Join(pullDir, filepath.FromSlash(name)) + got, err := os.ReadFile(extracted) + if err != nil { + t.Errorf("expected extracted file %s: %v", extracted, err) + continue + } + if !bytes.Equal(got, content) { + t.Errorf("extracted file %s does not match the pushed chart", name) + } + } +} + +// assertFilesEqual compares two files byte for byte. +func assertFilesEqual(t *testing.T, want, got string) { + t.Helper() + wantBytes, err := os.ReadFile(want) + if err != nil { + t.Fatalf("reading %s: %v", want, err) + } + gotBytes, err := os.ReadFile(got) + if err != nil { + t.Fatalf("reading %s: %v", got, err) + } + if !bytes.Equal(wantBytes, gotBytes) { + t.Errorf("pulled chart %s does not match the pushed chart %s (%d vs %d bytes)", + got, want, len(gotBytes), len(wantBytes)) + } +} + +// archiveFiles returns the regular files in a gzipped tar archive, keyed by +// their slash-separated path within the archive. +func archiveFiles(t *testing.T, path string) map[string][]byte { + t.Helper() + + f, err := os.Open(path) + if err != nil { + t.Fatalf("opening %s: %v", path, err) + } + defer f.Close() + + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatalf("reading %s as gzip: %v", path, err) + } + defer gz.Close() + + files := map[string][]byte{} + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("reading %s as tar: %v", path, err) + } + if hdr.Typeflag != tar.TypeReg { + continue + } + content, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("reading %s from %s: %v", hdr.Name, path, err) + } + files[hdr.Name] = content + } + return files +} + +// deleteNamespace removes a namespace created by the test run. It is best +// effort in the sense that it reports a failure rather than aborting the test, +// but it does not silently leave the namespace behind. +func deleteNamespace(t *testing.T, namespace string) { + t.Helper() + + cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), + &clientcmd.ConfigOverrides{}, + ).ClientConfig() + if err != nil { + t.Errorf("building kube client config to delete namespace %s: %v", namespace, err) + return + } + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + t.Errorf("building kube client to delete namespace %s: %v", namespace, err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + t.Logf("deleting namespace %s", namespace) + if err := client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}); err != nil { + if apierrors.IsNotFound(err) { + return + } + t.Errorf("deleting namespace %s: %v", namespace, err) + return + } + + // Deletion is asynchronous. Wait for the namespace to actually go away so + // that a namespace wedged in Terminating is reported rather than quietly + // accumulating in the cluster. + err = wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { + _, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + }) + if err != nil { + t.Errorf("waiting for namespace %s to be deleted: %v", namespace, err) + } +} From 33e8d8011bfd4617902336b8a4ad9ca430368837 Mon Sep 17 00:00:00 2001 From: Terry Howe Date: Mon, 21 Sep 2026 12:46:22 -0600 Subject: [PATCH 4/7] test: run OCI e2e tests across registry implementations The suite has no registry-specific behavior, so covering a new registry is a matter of pointing it at one. Add the CI wiring to do that, and make the repository paths suitable for shared registries. - Repository paths are now stable rather than per-run. The fixtures are immutable, so re-running overwrites each tag with identical content. This bounds the repositories a shared registry accumulates and lets registries that require repositories to exist before a push (ECR) have them created ahead of time. HELM_E2E_ISOLATE restores per-run paths. - Add HELM_E2E_REPO to set the repository path under the registry. - Add e2e.yml: local authenticated registry plus kind, no secrets, so it runs on pull requests from forks. - Add e2e-registries.yml: nightly matrix over GHCR, Quay, and ECR with fail-fast disabled. A registry with no secrets configured is skipped, so they can be wired up one at a time. Signed-off-by: Terry Howe --- .github/workflows/e2e-registries.yml | 143 +++++++++++++++++++++++++++ .github/workflows/e2e.yml | 86 ++++++++++++++++ test/e2e/README.md | 43 +++++++- test/e2e/helper_test.go | 41 +++++++- test/e2e/oci_test.go | 6 +- 5 files changed, 310 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/e2e-registries.yml create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/e2e-registries.yml b/.github/workflows/e2e-registries.yml new file mode 100644 index 000000000..d60058422 --- /dev/null +++ b/.github/workflows/e2e-registries.yml @@ -0,0 +1,143 @@ +name: e2e-registries + +# Runs the OCI end-to-end suite against real registry implementations, to catch +# changes that break a particular registry rather than OCI in general. +# +# This workflow needs credentials, so it deliberately does not run on +# pull_request: fork PRs cannot read secrets and would fail for every outside +# contributor. Secret-free coverage against a local registry runs per-PR in +# e2e.yml. Each registry is a separate matrix leg with fail-fast disabled, so +# one registry being down does not hide the results of the others. +# +# Required secrets, per registry. A leg whose secrets are absent is skipped +# rather than failed, so registries can be wired up one at a time. +# +# GHCR HELM_E2E_GHCR_REGISTRY e.g. ghcr.io/helm +# HELM_E2E_GHCR_USERNAME +# HELM_E2E_GHCR_TOKEN PAT with write:packages +# +# Quay HELM_E2E_QUAY_REGISTRY e.g. quay.io/helm +# HELM_E2E_QUAY_USERNAME robot account, e.g. helm+e2e +# HELM_E2E_QUAY_TOKEN robot account token +# +# ECR HELM_E2E_ECR_REGISTRY e.g. 111122223333.dkr.ecr.us-east-1.amazonaws.com +# HELM_E2E_ECR_REGION e.g. us-east-1 +# HELM_E2E_ECR_ACCESS_KEY_ID +# HELM_E2E_ECR_SECRET_ACCESS_KEY +# +# ECR is the odd one out in two ways: it mints a short-lived password rather +# than using a static one, and it does not create repositories on push, so the +# repositories are created below before the tests run. + +on: + schedule: + # Nightly, after most merges have landed. + - cron: "0 6 * * *" + workflow_dispatch: + +permissions: + contents: read + +env: + KIND_VERSION: v0.31.0 + # Repository path under each registry. Stable rather than per-run, so the + # set of repositories stays bounded and can be pre-created for ECR. + HELM_E2E_REPO: helm-e2e + +jobs: + registries: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + name: [ghcr, quay, ecr] + name: ${{ matrix.name }} + steps: + # Secrets cannot be referenced from strategy.matrix, so every registry's + # secrets are bound here and the matrix leg selects among them. They are + # read as environment variables rather than interpolated into the script, + # so no credential becomes shell source. + - name: Resolve registry configuration + id: config + env: + MATRIX_NAME: ${{ matrix.name }} + GHCR_REGISTRY: ${{ secrets.HELM_E2E_GHCR_REGISTRY }} + GHCR_USERNAME: ${{ secrets.HELM_E2E_GHCR_USERNAME }} + GHCR_PASSWORD: ${{ secrets.HELM_E2E_GHCR_TOKEN }} + QUAY_REGISTRY: ${{ secrets.HELM_E2E_QUAY_REGISTRY }} + QUAY_USERNAME: ${{ secrets.HELM_E2E_QUAY_USERNAME }} + QUAY_PASSWORD: ${{ secrets.HELM_E2E_QUAY_TOKEN }} + ECR_REGISTRY: ${{ secrets.HELM_E2E_ECR_REGISTRY }} + # ECR authenticates as the fixed user "AWS" with a token minted below. + ECR_USERNAME: AWS + ECR_PASSWORD: "" + run: | + set -euo pipefail + prefix="$(printf '%s' "${MATRIX_NAME}" | tr '[:lower:]' '[:upper:]')" + registry="${prefix}_REGISTRY" + username="${prefix}_USERNAME" + password="${prefix}_PASSWORD" + + # A registry with no secrets set is skipped rather than failed, so + # registries can be wired up one at a time. + if [ -z "${!registry:-}" ]; then + echo "no registry secret set for ${MATRIX_NAME}, skipping this leg" + echo "configured=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + { + echo "HELM_E2E_REGISTRY=${!registry}" + echo "HELM_E2E_USERNAME=${!username:-}" + } >> "$GITHUB_ENV" + if [ -n "${!password:-}" ]; then + echo "HELM_E2E_PASSWORD=${!password}" >> "$GITHUB_ENV" + fi + echo "configured=true" >> "$GITHUB_OUTPUT" + + - name: Checkout source code + if: steps.config.outputs.configured == 'true' + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # pin@v6.0.0 + - name: Add variables to environment file + if: steps.config.outputs.configured == 'true' + run: cat ".github/env" >> "$GITHUB_ENV" + - name: Setup Go + if: steps.config.outputs.configured == 'true' + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # pin@6.1.0 + with: + go-version: '${{ env.GOLANG_VERSION }}' + check-latest: true + + # ECR issues a short-lived authorization token rather than accepting a + # long-lived secret, and does not create repositories on push. + - name: Prepare ECR + if: steps.config.outputs.configured == 'true' && matrix.name == 'ecr' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.HELM_E2E_ECR_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.HELM_E2E_ECR_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.HELM_E2E_ECR_REGION }} + run: | + set -euo pipefail + # Helm appends the chart name to HELM_E2E_REPO, so each fixture needs + # its own repository. Keep this list in step with + # test/e2e/testdata/testcharts. + for chart in test compressedchart compressedchart-with-hyphens unicode-chart; do + aws ecr describe-repositories --repository-names "${HELM_E2E_REPO}/${chart}" >/dev/null 2>&1 \ + || aws ecr create-repository --repository-name "${HELM_E2E_REPO}/${chart}" >/dev/null + done + token="$(aws ecr get-login-password)" + echo "::add-mask::${token}" + echo "HELM_E2E_PASSWORD=${token}" >> "$GITHUB_ENV" + + - name: Create a kind cluster + if: steps.config.outputs.configured == 'true' + run: | + set -euo pipefail + go install "sigs.k8s.io/kind@${KIND_VERSION}" + kind create cluster --name helm-e2e --wait 120s + + - name: Run end-to-end tests + if: steps.config.outputs.configured == 'true' + env: + HELM_E2E_KUBERNETES: "true" + run: make test-e2e diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..da64696fe --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,86 @@ +name: e2e + +on: + push: + branches: + - "main" + - "dev-v3" + - "release-**" + pull_request: + branches: + - "main" + - "dev-v3" + workflow_dispatch: + +permissions: + contents: read + +env: + KIND_VERSION: v0.31.0 + REGISTRY_IMAGE: registry:2.8.3 + HTPASSWD_IMAGE: httpd:2.4-alpine + +jobs: + # Runs the OCI end-to-end suite against a local, authenticated registry and a + # kind cluster. Needs no secrets, so it runs on pull requests from forks. + # Cross-registry coverage lives in e2e-registries.yml. + local-registry: + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # pin@v6.0.0 + - name: Add variables to environment file + run: cat ".github/env" >> "$GITHUB_ENV" + - name: Setup Go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # pin@6.1.0 + with: + go-version: '${{ env.GOLANG_VERSION }}' + check-latest: true + + - name: Start an authenticated local registry + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/registry-auth" + docker run --rm --entrypoint htpasswd "${HTPASSWD_IMAGE}" \ + -Bbn e2euser e2epass > "${RUNNER_TEMP}/registry-auth/htpasswd" + docker run -d --name helm-e2e-registry -p 5000:5000 \ + -v "${RUNNER_TEMP}/registry-auth:/auth" \ + -e REGISTRY_AUTH=htpasswd \ + -e REGISTRY_AUTH_HTPASSWD_REALM=Registry \ + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ + "${REGISTRY_IMAGE}" + # Wait for the registry to answer before handing it to the tests. An + # unauthenticated /v2/ must be rejected, which also confirms that the + # htpasswd file was picked up and the invalid-credential test will + # have something to assert against. + for _ in $(seq 1 30); do + if [ "$(curl -s -o /dev/null -w '%{http_code}' http://localhost:5000/v2/)" = "401" ]; then + echo "registry is up and requires authentication" + exit 0 + fi + sleep 1 + done + echo "local registry did not come up with authentication enabled" + docker logs helm-e2e-registry + exit 1 + + - name: Create a kind cluster + run: | + set -euo pipefail + go install "sigs.k8s.io/kind@${KIND_VERSION}" + kind create cluster --name helm-e2e --wait 120s + kubectl cluster-info --context kind-helm-e2e + + - name: Run end-to-end tests + env: + HELM_E2E_REGISTRY: localhost:5000 + HELM_E2E_REPO: helm-e2e + HELM_E2E_USERNAME: e2euser + HELM_E2E_PASSWORD: e2epass + HELM_E2E_PLAIN_HTTP: "true" + HELM_E2E_KUBERNETES: "true" + run: make test-e2e + + - name: Collect registry logs on failure + if: failure() + run: docker logs helm-e2e-registry diff --git a/test/e2e/README.md b/test/e2e/README.md index 678cec038..ba3c3fd7d 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -36,6 +36,8 @@ auth-enabled registry to cover that path. | `HELM_E2E_REGISTRY` | yes | Registry namespace to push to, without a scheme (e.g. `ghcr.io/your-org`). | | `HELM_E2E_USERNAME` | yes | Username for the registry. | | `HELM_E2E_PASSWORD` | yes | Password or token for the registry. Passed to helm on stdin and never logged. | +| `HELM_E2E_REPO` | no | Repository path under the registry. Defaults to `helm-e2e`. | +| `HELM_E2E_ISOLATE` | no | Append a per-run suffix to the repository path. Off by default; see below. | | `HELM_E2E_BIN` | no | Path to a prebuilt helm binary. Defaults to building one from the working tree. | | `HELM_E2E_PLAIN_HTTP` | no | Set to use plain HTTP, for registries without TLS. | | `HELM_E2E_KUBERNETES` | no | Set to run the install test against the current kubecontext. Off by default because it mutates. | @@ -44,8 +46,45 @@ auth-enabled registry to cover that path. Because the `e2e` build tag is already an explicit opt-in, a missing required variable fails the test rather than silently skipping it. -Each run pushes to repositories suffixed with a random run ID, so concurrent -runs and repeated runs in a shared namespace do not collide. +## Repository layout + +Helm appends the chart name to the push target, so a run touches one repository +per fixture: + +``` +//test +//compressedchart +//compressedchart-with-hyphens +//unicode-chart +``` + +That set is stable by default. The fixtures are immutable, so re-running +overwrites each tag with byte-identical content: concurrent runs do not +interfere, a shared registry does not accumulate repositories over time, and +registries that require a repository to exist before a push (ECR) can have +these created ahead of time. + +Set `HELM_E2E_ISOLATE` to append a per-run suffix instead, which is useful when +several people share one registry namespace and you want a run to stand alone. +Note that stable paths stay safe only while every test pushes identical content +under a given tag; a test that pushes mutated content under a fixed tag would +need isolation. + +## Running against several registries + +The suite has no registry-specific behavior — everything comes from the +environment — so covering a new registry is a matter of pointing the same tests +at it. This is how `.github/workflows/e2e-registries.yml` exercises GHCR, Quay, +and ECR on a schedule, one matrix leg each, to catch changes that break a +particular registry implementation rather than OCI in general. + +Two registry differences are worth knowing about: + +- A default `registry:2` serves anonymous access, so the invalid-credential + test skips itself there, as described above. +- ECR does not create repositories on push and issues a short-lived token + rather than accepting a static password. The workflow creates the four + repositories listed above and mints a token before running the tests. The Kubernetes test deletes the namespace it creates. If you supply `HELM_E2E_NAMESPACE`, that namespace is left in place and only the releases are diff --git a/test/e2e/helper_test.go b/test/e2e/helper_test.go index 91ed186ac..7d0c1cf84 100644 --- a/test/e2e/helper_test.go +++ b/test/e2e/helper_test.go @@ -51,6 +51,15 @@ const ( // envPlainHTTP requests plain HTTP for registries without TLS, which is // useful when testing against a local registry. envPlainHTTP = "HELM_E2E_PLAIN_HTTP" + // envRepo is the repository path under the registry that charts are + // pushed to. Helm appends the chart name, so this is a prefix shared by + // every fixture. + envRepo = "HELM_E2E_REPO" + // envIsolate appends a per-run suffix to the repository path. It is off by + // default so that the set of repositories a run touches is stable and can + // be created ahead of time on registries that do not create repositories + // on push. + envIsolate = "HELM_E2E_ISOLATE" // envKubernetes opts in to the tests that install charts into a real // Kubernetes cluster using the ambient kubeconfig. envKubernetes = "HELM_E2E_KUBERNETES" @@ -70,6 +79,10 @@ type harness struct { password string // plainHTTP is true when the registry should be reached over HTTP. plainHTTP bool + // repoBase is the repository path charts are pushed under. + repoBase string + // isolate appends runID to repoBase. + isolate bool // configPath is the registry credential file used for this run. configPath string // runID isolates the repositories written by a single test run so that @@ -88,6 +101,8 @@ func newHarness(t *testing.T) *harness { username: requireEnv(t, envUsername), password: requireEnv(t, envPassword), plainHTTP: os.Getenv(envPlainHTTP) != "", + repoBase: envOr(envRepo, "helm-e2e"), + isolate: os.Getenv(envIsolate) != "", helmBin: helmBinary(t), runID: runID(t), } @@ -146,10 +161,28 @@ func (h *harness) registryHost() string { return host } -// repo returns an isolated repository path for this run, so that a shared -// registry namespace can serve many runs without interference. -func (h *harness) repo(name string) string { - return fmt.Sprintf("%s/%s-%s", h.registry, name, h.runID) +// repo returns the repository path charts are pushed to. Helm appends the +// chart name, so a run touches one repository per fixture. +// +// The path is stable by default. Re-pushing a fixture overwrites the same tag +// with byte-identical content, so concurrent runs do not interfere, the set of +// repositories stays bounded, and registries that require repositories to +// exist before a push (ECR) can have them created ahead of time. Set +// HELM_E2E_ISOLATE to give a run its own repositories instead. +func (h *harness) repo() string { + if h.isolate { + return fmt.Sprintf("%s/%s-%s", h.registry, h.repoBase, h.runID) + } + return fmt.Sprintf("%s/%s", h.registry, h.repoBase) +} + +// envOr returns the value of the named environment variable, or def when it is +// unset. +func envOr(name, def string) string { + if v := os.Getenv(name); v != "" { + return v + } + return def } // ref returns a full OCI reference for a chart within an isolated repository. diff --git a/test/e2e/oci_test.go b/test/e2e/oci_test.go index 92699fcbe..076419cda 100644 --- a/test/e2e/oci_test.go +++ b/test/e2e/oci_test.go @@ -45,7 +45,7 @@ func TestOCIRegistryPushPull(t *testing.T) { h := newHarness(t) h.mustLogin(t) - repo := h.repo("push-pull") + repo := h.repo() contentCache := t.TempDir() tests := []struct { @@ -142,7 +142,7 @@ func TestOCIRegistryInvalidCredentials(t *testing.T) { // locally for want of any credential at all. writeRegistryCredential(t, h.registryConfig(t), h.registryHost(), h.username, "invalid-"+h.runID) - out, err = h.helm(t, "push", chart(t, "test-0.1.0.tgz"), "oci://"+h.repo("auth-failure")) + out, err = h.helm(t, "push", chart(t, "test-0.1.0.tgz"), "oci://"+h.repo()) if err == nil { t.Fatal("expected push to fail with an invalid credential, but it succeeded") } @@ -216,7 +216,7 @@ func TestOCIRegistryInstallToKubernetes(t *testing.T) { namespace = "helm-e2e-" + h.runID t.Cleanup(func() { deleteNamespace(t, namespace) }) } - repo := h.repo("install") + repo := h.repo() tests := []struct { name string From 4565758ff817ad26e7d5ac17998a0d85c842b0cd Mon Sep 17 00:00:00 2001 From: Terry Howe Date: Tue, 22 Sep 2026 08:15:38 -0600 Subject: [PATCH 5/7] test: address review feedback on e2e harness and workflow - Build the helm binary by relative package path instead of the v4 module path, so the harness also works on branches where the module carries a different major version suffix. - Scope Kubernetes release names to the run. A caller-supplied HELM_E2E_NAMESPACE may already hold releases, and a fixed name could collide with one that cleanup would then uninstall. - Fail a partially configured registry leg in the nightly workflow instead of reporting it as configured and failing later in the harness or the AWS CLI. A leg with no registry set is still skipped. Signed-off-by: Terry Howe --- .github/workflows/e2e-registries.yml | 34 ++++++++++++++++++++++++---- test/e2e/helper_test.go | 4 +++- test/e2e/oci_test.go | 16 +++++++++---- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e-registries.yml b/.github/workflows/e2e-registries.yml index d60058422..130409282 100644 --- a/.github/workflows/e2e-registries.yml +++ b/.github/workflows/e2e-registries.yml @@ -9,8 +9,10 @@ name: e2e-registries # e2e.yml. Each registry is a separate matrix leg with fail-fast disabled, so # one registry being down does not hide the results of the others. # -# Required secrets, per registry. A leg whose secrets are absent is skipped -# rather than failed, so registries can be wired up one at a time. +# Required secrets, per registry. A leg whose registry secret is absent is +# skipped entirely, so registries can be wired up one at a time. A leg whose +# registry is set but is missing any of its other secrets is a misconfiguration +# and fails loudly rather than being silently skipped. # # GHCR HELM_E2E_GHCR_REGISTRY e.g. ghcr.io/helm # HELM_E2E_GHCR_USERNAME @@ -71,6 +73,9 @@ jobs: # ECR authenticates as the fixed user "AWS" with a token minted below. ECR_USERNAME: AWS ECR_PASSWORD: "" + ECR_REGION: ${{ secrets.HELM_E2E_ECR_REGION }} + ECR_ACCESS_KEY_ID: ${{ secrets.HELM_E2E_ECR_ACCESS_KEY_ID }} + ECR_SECRET_ACCESS_KEY: ${{ secrets.HELM_E2E_ECR_SECRET_ACCESS_KEY }} run: | set -euo pipefail prefix="$(printf '%s' "${MATRIX_NAME}" | tr '[:lower:]' '[:upper:]')" @@ -78,14 +83,35 @@ jobs: username="${prefix}_USERNAME" password="${prefix}_PASSWORD" - # A registry with no secrets set is skipped rather than failed, so - # registries can be wired up one at a time. + # A registry with nothing configured is skipped, so registries can be + # wired up one at a time. if [ -z "${!registry:-}" ]; then echo "no registry secret set for ${MATRIX_NAME}, skipping this leg" echo "configured=false" >> "$GITHUB_OUTPUT" exit 0 fi + # A registry that is configured but incomplete is an error. Skipping + # it would quietly report success for a registry nobody is testing, + # and letting it through only fails later inside the test harness. + missing="" + case "${MATRIX_NAME}" in + ecr) + # ECR mints its password below, but needs AWS credentials to do so. + [ -n "${ECR_REGION:-}" ] || missing="${missing} HELM_E2E_ECR_REGION" + [ -n "${ECR_ACCESS_KEY_ID:-}" ] || missing="${missing} HELM_E2E_ECR_ACCESS_KEY_ID" + [ -n "${ECR_SECRET_ACCESS_KEY:-}" ] || missing="${missing} HELM_E2E_ECR_SECRET_ACCESS_KEY" + ;; + *) + [ -n "${!username:-}" ] || missing="${missing} HELM_E2E_${prefix}_USERNAME" + [ -n "${!password:-}" ] || missing="${missing} HELM_E2E_${prefix}_TOKEN" + ;; + esac + if [ -n "${missing}" ]; then + echo "::error::${MATRIX_NAME} is partially configured; missing:${missing}" + exit 1 + fi + { echo "HELM_E2E_REGISTRY=${!registry}" echo "HELM_E2E_USERNAME=${!username:-}" diff --git a/test/e2e/helper_test.go b/test/e2e/helper_test.go index 7d0c1cf84..82770323a 100644 --- a/test/e2e/helper_test.go +++ b/test/e2e/helper_test.go @@ -136,8 +136,10 @@ func helmBinary(t *testing.T) string { return abs } + // Build by relative package path rather than module path, so this works + // on any branch regardless of the module's major version suffix. bin := filepath.Join(t.TempDir(), "helm") - build := exec.Command("go", "build", "-o", bin, "helm.sh/helm/v4/cmd/helm") + build := exec.Command("go", "build", "-o", bin, filepath.Join("..", "..", "cmd", "helm")) if out, err := build.CombinedOutput(); err != nil { t.Fatalf("building helm: %v\n%s", err, out) } diff --git a/test/e2e/oci_test.go b/test/e2e/oci_test.go index 076419cda..afd0e7daf 100644 --- a/test/e2e/oci_test.go +++ b/test/e2e/oci_test.go @@ -25,6 +25,7 @@ import ( "context" "encoding/base64" "encoding/json" + "fmt" "io" "os" "path/filepath" @@ -245,13 +246,18 @@ func TestOCIRegistryInstallToKubernetes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { h.mustHelm(t, "push", chart(t, tt.chart), "oci://"+repo) + // Scope the release name to this run. A caller-supplied namespace + // may already hold releases, and a fixed name could collide with + // one, which cleanup would then uninstall. + releaseName := fmt.Sprintf("%s-%s", tt.releaseName, h.runID) + t.Cleanup(func() { - if out, err := h.helm(t, "uninstall", tt.releaseName, "--namespace", namespace, "--ignore-not-found", "--wait"); err != nil { - t.Errorf("uninstalling %s failed: %v\n%s", tt.releaseName, err, out) + if out, err := h.helm(t, "uninstall", releaseName, "--namespace", namespace, "--ignore-not-found", "--wait"); err != nil { + t.Errorf("uninstalling %s failed: %v\n%s", releaseName, err, out) } }) - h.mustHelm(t, "install", tt.releaseName, h.ref(repo, tt.chartName, ""), + h.mustHelm(t, "install", releaseName, h.ref(repo, tt.chartName, ""), "--version", tt.chartVersion, "--namespace", namespace, "--create-namespace", @@ -261,9 +267,9 @@ func TestOCIRegistryInstallToKubernetes(t *testing.T) { // Read the release back from cluster storage to confirm the // install really reached the API server. - out := h.mustHelm(t, "status", tt.releaseName, "--namespace", namespace, "--output", "json") + out := h.mustHelm(t, "status", releaseName, "--namespace", namespace, "--output", "json") if !strings.Contains(out, `"status":"deployed"`) { - t.Errorf("expected release %s to be deployed, got:\n%s", tt.releaseName, out) + t.Errorf("expected release %s to be deployed, got:\n%s", releaseName, out) } }) } From b7bf07cdfd69f6b7b68739ea36e0ea077c8cb04e Mon Sep 17 00:00:00 2001 From: Terry Howe Date: Tue, 22 Sep 2026 11:11:48 -0600 Subject: [PATCH 6/7] test: honor HELM_KUBECONTEXT when deleting the e2e namespace Helm subprocesses inherit HELM_KUBECONTEXT from the environment, but the cleanup client loaded the kubeconfig's current-context. When the two differ, releases are installed into one cluster while the namespace delete is sent to another, where it returns NotFound and is treated as success, leaking the namespace it was meant to remove. Apply the same context override when building the cleanup client. Signed-off-by: Terry Howe --- test/e2e/helper_test.go | 4 ++++ test/e2e/oci_test.go | 13 ++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/test/e2e/helper_test.go b/test/e2e/helper_test.go index 82770323a..f869224c9 100644 --- a/test/e2e/helper_test.go +++ b/test/e2e/helper_test.go @@ -65,6 +65,10 @@ const ( envKubernetes = "HELM_E2E_KUBERNETES" // envNamespace is the namespace the Kubernetes tests install into. envNamespace = "HELM_E2E_NAMESPACE" + // envHelmKubeContext is helm's own kubecontext variable. The tests do not + // set it, but helm subprocesses inherit it, so cleanup performed through + // the Kubernetes API has to honor it too. + envHelmKubeContext = "HELM_KUBECONTEXT" ) // harness carries the resolved configuration shared by the end-to-end tests. diff --git a/test/e2e/oci_test.go b/test/e2e/oci_test.go index afd0e7daf..1a841f8cb 100644 --- a/test/e2e/oci_test.go +++ b/test/e2e/oci_test.go @@ -370,9 +370,20 @@ func archiveFiles(t *testing.T, path string) map[string][]byte { func deleteNamespace(t *testing.T, namespace string) { t.Helper() + // Select the same context the helm subprocesses used. They inherit + // HELM_KUBECONTEXT from the environment, so without this override the + // releases would be installed into one cluster while the namespace was + // deleted from whichever cluster the kubeconfig's current-context names. + // A delete against the wrong cluster returns NotFound, which would look + // like success while leaking the namespace. + overrides := &clientcmd.ConfigOverrides{} + if kubeContext := os.Getenv(envHelmKubeContext); kubeContext != "" { + overrides.CurrentContext = kubeContext + } + cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( clientcmd.NewDefaultClientConfigLoadingRules(), - &clientcmd.ConfigOverrides{}, + overrides, ).ClientConfig() if err != nil { t.Errorf("building kube client config to delete namespace %s: %v", namespace, err) From 17440df5265e43331015c463abf5de3644352357 Mon Sep 17 00:00:00 2001 From: Terry Howe Date: Tue, 22 Sep 2026 12:47:35 -0600 Subject: [PATCH 7/7] test: build e2e cleanup client from helm's own settings Overriding only HELM_KUBECONTEXT still left the cleanup client able to diverge from the helm subprocesses, which also honor HELM_KUBEAPISERVER, HELM_KUBETOKEN, HELM_KUBECAFILE, HELM_KUBETLS_SERVER_NAME and HELM_KUBEINSECURE_SKIP_TLS_VERIFY. Any of those could send the namespace delete to a different endpoint, where NotFound reads as success and the namespace leaks. Resolve the client through cli.New().RESTClientGetter() instead, so it is built exactly the way the subprocesses build theirs and stays in step without mirroring each variable by hand. Signed-off-by: Terry Howe --- test/e2e/helper_test.go | 4 ---- test/e2e/oci_test.go | 27 +++++++++++---------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/test/e2e/helper_test.go b/test/e2e/helper_test.go index f869224c9..82770323a 100644 --- a/test/e2e/helper_test.go +++ b/test/e2e/helper_test.go @@ -65,10 +65,6 @@ const ( envKubernetes = "HELM_E2E_KUBERNETES" // envNamespace is the namespace the Kubernetes tests install into. envNamespace = "HELM_E2E_NAMESPACE" - // envHelmKubeContext is helm's own kubecontext variable. The tests do not - // set it, but helm subprocesses inherit it, so cleanup performed through - // the Kubernetes API has to honor it too. - envHelmKubeContext = "HELM_KUBECONTEXT" ) // harness carries the resolved configuration shared by the end-to-end tests. diff --git a/test/e2e/oci_test.go b/test/e2e/oci_test.go index 1a841f8cb..971d0d986 100644 --- a/test/e2e/oci_test.go +++ b/test/e2e/oci_test.go @@ -37,7 +37,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" + + "helm.sh/helm/v4/pkg/cli" ) // TestOCIRegistryPushPull pushes chart archives to the configured OCI registry @@ -370,21 +371,15 @@ func archiveFiles(t *testing.T, path string) map[string][]byte { func deleteNamespace(t *testing.T, namespace string) { t.Helper() - // Select the same context the helm subprocesses used. They inherit - // HELM_KUBECONTEXT from the environment, so without this override the - // releases would be installed into one cluster while the namespace was - // deleted from whichever cluster the kubeconfig's current-context names. - // A delete against the wrong cluster returns NotFound, which would look - // like success while leaking the namespace. - overrides := &clientcmd.ConfigOverrides{} - if kubeContext := os.Getenv(envHelmKubeContext); kubeContext != "" { - overrides.CurrentContext = kubeContext - } - - cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - clientcmd.NewDefaultClientConfigLoadingRules(), - overrides, - ).ClientConfig() + // Resolve the cluster exactly the way the helm subprocesses do. They read + // their Kubernetes configuration from the environment (HELM_KUBECONTEXT, + // HELM_KUBEAPISERVER, HELM_KUBETOKEN, HELM_KUBECAFILE and friends), so + // building this client any other way risks installing releases into one + // cluster and sending the namespace delete to another. Such a delete + // returns NotFound, which would look like success while leaking the + // namespace. Going through helm's own settings keeps the two in step + // without having to mirror each variable here. + cfg, err := cli.New().RESTClientGetter().ToRESTConfig() if err != nil { t.Errorf("building kube client config to delete namespace %s: %v", namespace, err) return