diff --git a/.github/workflows/e2e-registries.yml b/.github/workflows/e2e-registries.yml new file mode 100644 index 000000000..130409282 --- /dev/null +++ b/.github/workflows/e2e-registries.yml @@ -0,0 +1,169 @@ +name: e2e-registries + +# Runs the OCI end-to-end suite against real registry implementations, to catch +# changes that break a particular registry rather than OCI in general. +# +# This workflow needs credentials, so it deliberately does not run on +# pull_request: fork PRs cannot read secrets and would fail for every outside +# contributor. Secret-free coverage against a local registry runs per-PR in +# e2e.yml. Each registry is a separate matrix leg with fail-fast disabled, so +# one registry being down does not hide the results of the others. +# +# Required secrets, per registry. A leg whose registry secret is absent is +# skipped entirely, so registries can be wired up one at a time. A leg whose +# registry is set but is missing any of its other secrets is a misconfiguration +# and fails loudly rather than being silently skipped. +# +# GHCR HELM_E2E_GHCR_REGISTRY e.g. ghcr.io/helm +# HELM_E2E_GHCR_USERNAME +# HELM_E2E_GHCR_TOKEN PAT with write:packages +# +# Quay HELM_E2E_QUAY_REGISTRY e.g. quay.io/helm +# HELM_E2E_QUAY_USERNAME robot account, e.g. helm+e2e +# HELM_E2E_QUAY_TOKEN robot account token +# +# ECR HELM_E2E_ECR_REGISTRY e.g. 111122223333.dkr.ecr.us-east-1.amazonaws.com +# HELM_E2E_ECR_REGION e.g. us-east-1 +# HELM_E2E_ECR_ACCESS_KEY_ID +# HELM_E2E_ECR_SECRET_ACCESS_KEY +# +# ECR is the odd one out in two ways: it mints a short-lived password rather +# than using a static one, and it does not create repositories on push, so the +# repositories are created below before the tests run. + +on: + schedule: + # Nightly, after most merges have landed. + - cron: "0 6 * * *" + workflow_dispatch: + +permissions: + contents: read + +env: + KIND_VERSION: v0.31.0 + # Repository path under each registry. Stable rather than per-run, so the + # set of repositories stays bounded and can be pre-created for ECR. + HELM_E2E_REPO: helm-e2e + +jobs: + registries: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + name: [ghcr, quay, ecr] + name: ${{ matrix.name }} + steps: + # Secrets cannot be referenced from strategy.matrix, so every registry's + # secrets are bound here and the matrix leg selects among them. They are + # read as environment variables rather than interpolated into the script, + # so no credential becomes shell source. + - name: Resolve registry configuration + id: config + env: + MATRIX_NAME: ${{ matrix.name }} + GHCR_REGISTRY: ${{ secrets.HELM_E2E_GHCR_REGISTRY }} + GHCR_USERNAME: ${{ secrets.HELM_E2E_GHCR_USERNAME }} + GHCR_PASSWORD: ${{ secrets.HELM_E2E_GHCR_TOKEN }} + QUAY_REGISTRY: ${{ secrets.HELM_E2E_QUAY_REGISTRY }} + QUAY_USERNAME: ${{ secrets.HELM_E2E_QUAY_USERNAME }} + QUAY_PASSWORD: ${{ secrets.HELM_E2E_QUAY_TOKEN }} + ECR_REGISTRY: ${{ secrets.HELM_E2E_ECR_REGISTRY }} + # ECR authenticates as the fixed user "AWS" with a token minted below. + ECR_USERNAME: AWS + ECR_PASSWORD: "" + ECR_REGION: ${{ secrets.HELM_E2E_ECR_REGION }} + ECR_ACCESS_KEY_ID: ${{ secrets.HELM_E2E_ECR_ACCESS_KEY_ID }} + ECR_SECRET_ACCESS_KEY: ${{ secrets.HELM_E2E_ECR_SECRET_ACCESS_KEY }} + run: | + set -euo pipefail + prefix="$(printf '%s' "${MATRIX_NAME}" | tr '[:lower:]' '[:upper:]')" + registry="${prefix}_REGISTRY" + username="${prefix}_USERNAME" + password="${prefix}_PASSWORD" + + # A registry with nothing configured is skipped, so registries can be + # wired up one at a time. + if [ -z "${!registry:-}" ]; then + echo "no registry secret set for ${MATRIX_NAME}, skipping this leg" + echo "configured=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # A registry that is configured but incomplete is an error. Skipping + # it would quietly report success for a registry nobody is testing, + # and letting it through only fails later inside the test harness. + missing="" + case "${MATRIX_NAME}" in + ecr) + # ECR mints its password below, but needs AWS credentials to do so. + [ -n "${ECR_REGION:-}" ] || missing="${missing} HELM_E2E_ECR_REGION" + [ -n "${ECR_ACCESS_KEY_ID:-}" ] || missing="${missing} HELM_E2E_ECR_ACCESS_KEY_ID" + [ -n "${ECR_SECRET_ACCESS_KEY:-}" ] || missing="${missing} HELM_E2E_ECR_SECRET_ACCESS_KEY" + ;; + *) + [ -n "${!username:-}" ] || missing="${missing} HELM_E2E_${prefix}_USERNAME" + [ -n "${!password:-}" ] || missing="${missing} HELM_E2E_${prefix}_TOKEN" + ;; + esac + if [ -n "${missing}" ]; then + echo "::error::${MATRIX_NAME} is partially configured; missing:${missing}" + exit 1 + fi + + { + echo "HELM_E2E_REGISTRY=${!registry}" + echo "HELM_E2E_USERNAME=${!username:-}" + } >> "$GITHUB_ENV" + if [ -n "${!password:-}" ]; then + echo "HELM_E2E_PASSWORD=${!password}" >> "$GITHUB_ENV" + fi + echo "configured=true" >> "$GITHUB_OUTPUT" + + - name: Checkout source code + if: steps.config.outputs.configured == 'true' + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # pin@v6.0.0 + - name: Add variables to environment file + if: steps.config.outputs.configured == 'true' + run: cat ".github/env" >> "$GITHUB_ENV" + - name: Setup Go + if: steps.config.outputs.configured == 'true' + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # pin@6.1.0 + with: + go-version: '${{ env.GOLANG_VERSION }}' + check-latest: true + + # ECR issues a short-lived authorization token rather than accepting a + # long-lived secret, and does not create repositories on push. + - name: Prepare ECR + if: steps.config.outputs.configured == 'true' && matrix.name == 'ecr' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.HELM_E2E_ECR_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.HELM_E2E_ECR_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.HELM_E2E_ECR_REGION }} + run: | + set -euo pipefail + # Helm appends the chart name to HELM_E2E_REPO, so each fixture needs + # its own repository. Keep this list in step with + # test/e2e/testdata/testcharts. + for chart in test compressedchart compressedchart-with-hyphens unicode-chart; do + aws ecr describe-repositories --repository-names "${HELM_E2E_REPO}/${chart}" >/dev/null 2>&1 \ + || aws ecr create-repository --repository-name "${HELM_E2E_REPO}/${chart}" >/dev/null + done + token="$(aws ecr get-login-password)" + echo "::add-mask::${token}" + echo "HELM_E2E_PASSWORD=${token}" >> "$GITHUB_ENV" + + - name: Create a kind cluster + if: steps.config.outputs.configured == 'true' + run: | + set -euo pipefail + go install "sigs.k8s.io/kind@${KIND_VERSION}" + kind create cluster --name helm-e2e --wait 120s + + - name: Run end-to-end tests + if: steps.config.outputs.configured == 'true' + env: + HELM_E2E_KUBERNETES: "true" + run: make test-e2e diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..da64696fe --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,86 @@ +name: e2e + +on: + push: + branches: + - "main" + - "dev-v3" + - "release-**" + pull_request: + branches: + - "main" + - "dev-v3" + workflow_dispatch: + +permissions: + contents: read + +env: + KIND_VERSION: v0.31.0 + REGISTRY_IMAGE: registry:2.8.3 + HTPASSWD_IMAGE: httpd:2.4-alpine + +jobs: + # Runs the OCI end-to-end suite against a local, authenticated registry and a + # kind cluster. Needs no secrets, so it runs on pull requests from forks. + # Cross-registry coverage lives in e2e-registries.yml. + local-registry: + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # pin@v6.0.0 + - name: Add variables to environment file + run: cat ".github/env" >> "$GITHUB_ENV" + - name: Setup Go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # pin@6.1.0 + with: + go-version: '${{ env.GOLANG_VERSION }}' + check-latest: true + + - name: Start an authenticated local registry + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/registry-auth" + docker run --rm --entrypoint htpasswd "${HTPASSWD_IMAGE}" \ + -Bbn e2euser e2epass > "${RUNNER_TEMP}/registry-auth/htpasswd" + docker run -d --name helm-e2e-registry -p 5000:5000 \ + -v "${RUNNER_TEMP}/registry-auth:/auth" \ + -e REGISTRY_AUTH=htpasswd \ + -e REGISTRY_AUTH_HTPASSWD_REALM=Registry \ + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ + "${REGISTRY_IMAGE}" + # Wait for the registry to answer before handing it to the tests. An + # unauthenticated /v2/ must be rejected, which also confirms that the + # htpasswd file was picked up and the invalid-credential test will + # have something to assert against. + for _ in $(seq 1 30); do + if [ "$(curl -s -o /dev/null -w '%{http_code}' http://localhost:5000/v2/)" = "401" ]; then + echo "registry is up and requires authentication" + exit 0 + fi + sleep 1 + done + echo "local registry did not come up with authentication enabled" + docker logs helm-e2e-registry + exit 1 + + - name: Create a kind cluster + run: | + set -euo pipefail + go install "sigs.k8s.io/kind@${KIND_VERSION}" + kind create cluster --name helm-e2e --wait 120s + kubectl cluster-info --context kind-helm-e2e + + - name: Run end-to-end tests + env: + HELM_E2E_REGISTRY: localhost:5000 + HELM_E2E_REPO: helm-e2e + HELM_E2E_USERNAME: e2euser + HELM_E2E_PASSWORD: e2epass + HELM_E2E_PLAIN_HTTP: "true" + HELM_E2E_KUBERNETES: "true" + run: make test-e2e + + - name: Collect registry logs on failure + if: failure() + run: docker logs helm-e2e-registry diff --git a/Makefile b/Makefile index 5a9c792bb..108e58d28 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,15 @@ test-coverage: @echo "==> Running unit tests with coverage: $(PKG) <==" @ ./scripts/coverage.sh $(PKG) +# End-to-end tests run a real helm binary against a real OCI registry, and +# optionally a real Kubernetes cluster. They are behind the 'e2e' build tag and +# require configuration; see test/e2e/README.md. +.PHONY: test-e2e +test-e2e: + @echo + @echo "==> Running end-to-end tests <==" + go test $(GOFLAGS) -tags e2e -count=1 -v ./test/e2e/... + .PHONY: test-style test-style: @EXPECTED_VERSION=$$(grep GOLANGCI_LINT_VERSION .github/env | cut -d= -f2); \ diff --git a/pkg/cmd/testdata/testcharts/unicode-chart-0.1.0.tgz b/pkg/cmd/testdata/testcharts/unicode-chart-0.1.0.tgz new file mode 100644 index 000000000..9af1ce2c8 Binary files /dev/null and b/pkg/cmd/testdata/testcharts/unicode-chart-0.1.0.tgz differ diff --git a/pkg/cmd/testdata/testcharts/unicode-chart/Chart.yaml b/pkg/cmd/testdata/testcharts/unicode-chart/Chart.yaml new file mode 100644 index 000000000..9f5690f12 --- /dev/null +++ b/pkg/cmd/testdata/testcharts/unicode-chart/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Test chart with unicode Kröpke 日本語 中文 한글 +name: unicode-chart +version: 0.1.0 diff --git a/pkg/cmd/testdata/testcharts/unicode-chart/templates/NOTES.txt b/pkg/cmd/testdata/testcharts/unicode-chart/templates/NOTES.txt new file mode 100644 index 000000000..75ef6ee88 --- /dev/null +++ b/pkg/cmd/testdata/testcharts/unicode-chart/templates/NOTES.txt @@ -0,0 +1,3 @@ +Thank you for installing {{ .Chart.Name }}. + +This chart includes unicode: Kröpke 日本語 中文 한글 diff --git a/pkg/cmd/testdata/testcharts/unicode-chart/values.yaml b/pkg/cmd/testdata/testcharts/unicode-chart/values.yaml new file mode 100644 index 000000000..f5782aac6 --- /dev/null +++ b/pkg/cmd/testdata/testcharts/unicode-chart/values.yaml @@ -0,0 +1,2 @@ +# Default values for unicode-chart +replicaCount: 1 diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 000000000..ba3c3fd7d --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,91 @@ +# Helm end-to-end tests + +These tests exercise a real `helm` binary against real external systems: an OCI +registry, and optionally a Kubernetes cluster. They are guarded by the `e2e` +build tag, so they are never compiled or run by `make test`. + +Unlike the unit tests, nothing here is faked. The registry is a real registry, +and the Kubernetes test installs into whatever cluster your current kubecontext +points at. + +## Running + +```console +$ export HELM_E2E_REGISTRY=ghcr.io/your-org +$ export HELM_E2E_USERNAME=your-user +$ export HELM_E2E_PASSWORD=your-token +$ make test-e2e +``` + +Any OCI registry works, not just GHCR. To run against a local registry: + +```console +$ export HELM_E2E_REGISTRY=localhost:5000/charts +$ export HELM_E2E_PLAIN_HTTP=true +``` + +A default `registry:2` deployment serves anonymous access and accepts any +credential. Against such a registry there is no authentication to exercise, so +the invalid-credential test detects this and skips itself. Run against an +auth-enabled registry to cover that path. + +## Configuration + +| Variable | Required | Description | +| ---------------------- | -------- | ---------------------------------------------------------------------------------------------- | +| `HELM_E2E_REGISTRY` | yes | Registry namespace to push to, without a scheme (e.g. `ghcr.io/your-org`). | +| `HELM_E2E_USERNAME` | yes | Username for the registry. | +| `HELM_E2E_PASSWORD` | yes | Password or token for the registry. Passed to helm on stdin and never logged. | +| `HELM_E2E_REPO` | no | Repository path under the registry. Defaults to `helm-e2e`. | +| `HELM_E2E_ISOLATE` | no | Append a per-run suffix to the repository path. Off by default; see below. | +| `HELM_E2E_BIN` | no | Path to a prebuilt helm binary. Defaults to building one from the working tree. | +| `HELM_E2E_PLAIN_HTTP` | no | Set to use plain HTTP, for registries without TLS. | +| `HELM_E2E_KUBERNETES` | no | Set to run the install test against the current kubecontext. Off by default because it mutates. | +| `HELM_E2E_NAMESPACE` | no | Namespace for the Kubernetes test. Defaults to a unique `helm-e2e-` namespace. | + +Because the `e2e` build tag is already an explicit opt-in, a missing required +variable fails the test rather than silently skipping it. + +## Repository layout + +Helm appends the chart name to the push target, so a run touches one repository +per fixture: + +``` +//test +//compressedchart +//compressedchart-with-hyphens +//unicode-chart +``` + +That set is stable by default. The fixtures are immutable, so re-running +overwrites each tag with byte-identical content: concurrent runs do not +interfere, a shared registry does not accumulate repositories over time, and +registries that require a repository to exist before a push (ECR) can have +these created ahead of time. + +Set `HELM_E2E_ISOLATE` to append a per-run suffix instead, which is useful when +several people share one registry namespace and you want a run to stand alone. +Note that stable paths stay safe only while every test pushes identical content +under a given tag; a test that pushes mutated content under a fixed tag would +need isolation. + +## Running against several registries + +The suite has no registry-specific behavior — everything comes from the +environment — so covering a new registry is a matter of pointing the same tests +at it. This is how `.github/workflows/e2e-registries.yml` exercises GHCR, Quay, +and ECR on a schedule, one matrix leg each, to catch changes that break a +particular registry implementation rather than OCI in general. + +Two registry differences are worth knowing about: + +- A default `registry:2` serves anonymous access, so the invalid-credential + test skips itself there, as described above. +- ECR does not create repositories on push and issues a short-lived token + rather than accepting a static password. The workflow creates the four + repositories listed above and mints a token before running the tests. + +The Kubernetes test deletes the namespace it creates. If you supply +`HELM_E2E_NAMESPACE`, that namespace is left in place and only the releases are +uninstalled. diff --git a/test/e2e/helper_test.go b/test/e2e/helper_test.go new file mode 100644 index 000000000..82770323a --- /dev/null +++ b/test/e2e/helper_test.go @@ -0,0 +1,281 @@ +//go:build e2e + +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package e2e contains end-to-end tests that exercise a real helm binary +// against real external systems (an OCI registry, and optionally a Kubernetes +// cluster). +// +// These tests are excluded from the normal build by the "e2e" build tag and +// are never run by `make test`. Run them with `make test-e2e` after exporting +// the configuration described in the package README. +package e2e + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Environment variables used to configure the end-to-end tests. +const ( + // envRegistry is the registry namespace charts are pushed to, without a + // scheme, e.g. "ghcr.io/example-org" or "localhost:5000/charts". + envRegistry = "HELM_E2E_REGISTRY" + // envUsername is the username used to authenticate to envRegistry. + envUsername = "HELM_E2E_USERNAME" + // envPassword is the password or token used to authenticate to envRegistry. + envPassword = "HELM_E2E_PASSWORD" + // envHelmBin points at a prebuilt helm binary. When unset, one is built + // from the working tree. + envHelmBin = "HELM_E2E_BIN" + // envPlainHTTP requests plain HTTP for registries without TLS, which is + // useful when testing against a local registry. + envPlainHTTP = "HELM_E2E_PLAIN_HTTP" + // envRepo is the repository path under the registry that charts are + // pushed to. Helm appends the chart name, so this is a prefix shared by + // every fixture. + envRepo = "HELM_E2E_REPO" + // envIsolate appends a per-run suffix to the repository path. It is off by + // default so that the set of repositories a run touches is stable and can + // be created ahead of time on registries that do not create repositories + // on push. + envIsolate = "HELM_E2E_ISOLATE" + // envKubernetes opts in to the tests that install charts into a real + // Kubernetes cluster using the ambient kubeconfig. + envKubernetes = "HELM_E2E_KUBERNETES" + // envNamespace is the namespace the Kubernetes tests install into. + envNamespace = "HELM_E2E_NAMESPACE" +) + +// harness carries the resolved configuration shared by the end-to-end tests. +type harness struct { + // helmBin is an absolute path to the helm binary under test. + helmBin string + // registry is the registry namespace charts are pushed to. + registry string + // username and password authenticate to the registry. They are only ever + // passed to helm over stdin and are never logged. + username string + password string + // plainHTTP is true when the registry should be reached over HTTP. + plainHTTP bool + // repoBase is the repository path charts are pushed under. + repoBase string + // isolate appends runID to repoBase. + isolate bool + // configPath is the registry credential file used for this run. + configPath string + // runID isolates the repositories written by a single test run so that + // concurrent or repeated runs do not collide. + runID string +} + +// newHarness resolves the end-to-end configuration, failing the test when a +// required value is missing. Because the "e2e" build tag is an explicit opt-in, +// a missing setting is a configuration error rather than a reason to skip. +func newHarness(t *testing.T) *harness { + t.Helper() + + h := &harness{ + registry: requireEnv(t, envRegistry), + username: requireEnv(t, envUsername), + password: requireEnv(t, envPassword), + plainHTTP: os.Getenv(envPlainHTTP) != "", + repoBase: envOr(envRepo, "helm-e2e"), + isolate: os.Getenv(envIsolate) != "", + helmBin: helmBinary(t), + runID: runID(t), + } + return h +} + +// requireEnv returns the value of the named environment variable, failing the +// test with an actionable message when it is unset. +func requireEnv(t *testing.T, name string) string { + t.Helper() + v := os.Getenv(name) + if v == "" { + t.Fatalf("%s must be set to run the end-to-end tests; see test/e2e/README.md", name) + } + return v +} + +// helmBinary returns the helm binary to exercise, building one from the +// working tree when HELM_E2E_BIN is not set. +func helmBinary(t *testing.T) string { + t.Helper() + + if bin := os.Getenv(envHelmBin); bin != "" { + abs, err := filepath.Abs(bin) + if err != nil { + t.Fatalf("resolving %s=%q: %v", envHelmBin, bin, err) + } + if _, err := os.Stat(abs); err != nil { + t.Fatalf("%s=%q is not usable: %v", envHelmBin, bin, err) + } + return abs + } + + // Build by relative package path rather than module path, so this works + // on any branch regardless of the module's major version suffix. + bin := filepath.Join(t.TempDir(), "helm") + build := exec.Command("go", "build", "-o", bin, filepath.Join("..", "..", "cmd", "helm")) + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("building helm: %v\n%s", err, out) + } + return bin +} + +// runID returns a short random identifier unique to this test run. +func runID(t *testing.T) string { + t.Helper() + b := make([]byte, 4) + if _, err := rand.Read(b); err != nil { + t.Fatalf("generating run id: %v", err) + } + return hex.EncodeToString(b) +} + +// registryHost returns the host portion of the configured registry, which is +// what `helm registry login` expects. +func (h *harness) registryHost() string { + host, _, _ := strings.Cut(h.registry, "/") + return host +} + +// repo returns the repository path charts are pushed to. Helm appends the +// chart name, so a run touches one repository per fixture. +// +// The path is stable by default. Re-pushing a fixture overwrites the same tag +// with byte-identical content, so concurrent runs do not interfere, the set of +// repositories stays bounded, and registries that require repositories to +// exist before a push (ECR) can have them created ahead of time. Set +// HELM_E2E_ISOLATE to give a run its own repositories instead. +func (h *harness) repo() string { + if h.isolate { + return fmt.Sprintf("%s/%s-%s", h.registry, h.repoBase, h.runID) + } + return fmt.Sprintf("%s/%s", h.registry, h.repoBase) +} + +// envOr returns the value of the named environment variable, or def when it is +// unset. +func envOr(name, def string) string { + if v := os.Getenv(name); v != "" { + return v + } + return def +} + +// ref returns a full OCI reference for a chart within an isolated repository. +func (h *harness) ref(repo, chart, version string) string { + if version == "" { + return fmt.Sprintf("oci://%s/%s", repo, chart) + } + return fmt.Sprintf("oci://%s/%s:%s", repo, chart, version) +} + +// plainHTTPCommands are the helm subcommands that accept --plain-http. The +// flag is only appended for these so that the harness can still run commands +// such as `status` and `uninstall` against a plain HTTP registry. +var plainHTTPCommands = map[string]bool{"push": true, "pull": true, "install": true, "upgrade": true} + +// helm runs the helm binary with the given arguments and returns its combined +// output. Arguments are logged, so callers must never pass a credential. +func (h *harness) helm(t *testing.T, args ...string) (string, error) { + t.Helper() + if h.plainHTTP && len(args) > 0 && plainHTTPCommands[args[0]] { + args = append(args, "--plain-http") + } + t.Logf("helm %s", strings.Join(args, " ")) + cmd := exec.Command(h.helmBin, args...) + cmd.Env = append(os.Environ(), "HELM_REGISTRY_CONFIG="+h.registryConfig(t)) + out, err := cmd.CombinedOutput() + return string(out), err +} + +// mustHelm runs helm and fails the test if the command does not succeed. +func (h *harness) mustHelm(t *testing.T, args ...string) string { + t.Helper() + out, err := h.helm(t, args...) + if err != nil { + t.Fatalf("helm %s failed: %v\n%s", strings.Join(args, " "), err, out) + } + return out +} + +// registryConfig returns the path to this run's registry credential file. The +// file lives under a per-test temporary directory so credentials never leak +// into the developer's real helm configuration. +func (h *harness) registryConfig(t *testing.T) string { + t.Helper() + if h.configPath == "" { + h.configPath = filepath.Join(t.TempDir(), "registry-config.json") + } + return h.configPath +} + +// login authenticates to the configured registry. The password is written to +// helm's stdin rather than passed as a flag so it cannot appear in process +// listings or test output. +func (h *harness) login(t *testing.T, password string) (string, error) { + t.Helper() + args := []string{"registry", "login", h.registryHost(), "--username", h.username, "--password-stdin"} + if h.plainHTTP { + args = append(args, "--plain-http") + } + t.Logf("helm %s", strings.Join(args, " ")) + cmd := exec.Command(h.helmBin, args...) + cmd.Env = append(os.Environ(), "HELM_REGISTRY_CONFIG="+h.registryConfig(t)) + cmd.Stdin = strings.NewReader(password) + out, err := cmd.CombinedOutput() + return string(out), err +} + +// mustLogin authenticates with the configured credential and arranges for a +// logout once the test completes. +func (h *harness) mustLogin(t *testing.T) { + t.Helper() + out, err := h.login(t, h.password) + if err != nil { + t.Fatalf("registry login to %s failed: %v\n%s", h.registryHost(), err, out) + } + t.Cleanup(func() { + if out, err := h.helm(t, "registry", "logout", h.registryHost()); err != nil { + t.Logf("registry logout failed (ignored): %v\n%s", err, out) + } + }) +} + +// chart returns the path to a chart archive in this package's testdata. +func chart(t *testing.T, name string) string { + t.Helper() + path, err := filepath.Abs(filepath.Join("testdata", "testcharts", name)) + if err != nil { + t.Fatalf("resolving chart %q: %v", name, err) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("chart %q is missing: %v", name, err) + } + return path +} diff --git a/test/e2e/oci_test.go b/test/e2e/oci_test.go new file mode 100644 index 000000000..971d0d986 --- /dev/null +++ b/test/e2e/oci_test.go @@ -0,0 +1,418 @@ +//go:build e2e + +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + + "helm.sh/helm/v4/pkg/cli" +) + +// TestOCIRegistryPushPull pushes chart archives to the configured OCI registry +// and pulls them back, verifying that the artifact round-trips intact. +func TestOCIRegistryPushPull(t *testing.T) { + h := newHarness(t) + h.mustLogin(t) + + repo := h.repo() + contentCache := t.TempDir() + + tests := []struct { + name string + chart string + chartName string + chartVersion string + pullArgs []string + expectPullFile string + expectPullDir bool + }{ + { + name: "basic chart", + chart: "test-0.1.0.tgz", + chartName: "test", + chartVersion: "0.1.0", + expectPullFile: "test-0.1.0.tgz", + }, + { + name: "chart pulled with untar", + chart: "compressedchart-0.1.0.tgz", + chartName: "compressedchart", + chartVersion: "0.1.0", + pullArgs: []string{"--untar"}, + expectPullFile: "compressedchart", + expectPullDir: true, + }, + { + name: "chart name with hyphens", + chart: "compressedchart-with-hyphens-0.1.0.tgz", + chartName: "compressedchart-with-hyphens", + chartVersion: "0.1.0", + expectPullFile: "compressedchart-with-hyphens-0.1.0.tgz", + }, + { + name: "chart with unicode description", + chart: "unicode-chart-0.1.0.tgz", + chartName: "unicode-chart", + chartVersion: "0.1.0", + expectPullFile: "unicode-chart-0.1.0.tgz", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pullDir := t.TempDir() + + h.mustHelm(t, "push", chart(t, tt.chart), "oci://"+repo) + + pullArgs := append([]string{ + "pull", h.ref(repo, tt.chartName, tt.chartVersion), + "--destination", pullDir, + "--content-cache", contentCache, + }, tt.pullArgs...) + h.mustHelm(t, pullArgs...) + + pulled := filepath.Join(pullDir, tt.expectPullFile) + fi, err := os.Stat(pulled) + if err != nil { + t.Fatalf("expected pulled chart at %s: %v", pulled, err) + } + if fi.IsDir() != tt.expectPullDir { + t.Errorf("expected directory=%t, got directory=%t", tt.expectPullDir, fi.IsDir()) + } + + // Verify the round-trip actually returned the chart we pushed, + // not merely some artifact under the same tag. + assertChartMatches(t, chart(t, tt.chart), pullDir, tt.expectPullDir) + }) + } +} + +// TestOCIRegistryInvalidCredentials verifies that the registry rejects a bad +// credential with an authentication error, rather than any error at all. +func TestOCIRegistryInvalidCredentials(t *testing.T) { + h := newHarness(t) + + // Log in with a deliberately wrong password against the same registry and + // namespace the successful tests use, so the only variable is the + // credential itself. + out, err := h.login(t, "invalid-"+h.runID) + if err == nil { + // An anonymous registry, such as a default registry:2 deployment, + // accepts any credential. There is no authentication to exercise, so + // skip rather than report a failure the registry cannot produce. + t.Skipf("Skipping invalid credential test: %s accepts unauthenticated access", h.registryHost()) + } + if !isAuthError(out) { + t.Fatalf("expected an authentication failure, got a different error:\n%s", out) + } + + // Seed the credential store directly with the same wrong password, so the + // push reaches the registry and is rejected by it rather than failing + // locally for want of any credential at all. + writeRegistryCredential(t, h.registryConfig(t), h.registryHost(), h.username, "invalid-"+h.runID) + + out, err = h.helm(t, "push", chart(t, "test-0.1.0.tgz"), "oci://"+h.repo()) + if err == nil { + t.Fatal("expected push to fail with an invalid credential, but it succeeded") + } + if !isAuthError(out) { + t.Fatalf("expected an authentication failure, got a different error:\n%s", out) + } +} + +// writeRegistryCredential writes a docker-style credential file so a test can +// present a specific credential to the registry without going through +// `helm registry login`, which refuses to store one the registry rejects. +func writeRegistryCredential(t *testing.T, path, host, username, password string) { + t.Helper() + cfg := struct { + Auths map[string]struct { + Auth string `json:"auth"` + } `json:"auths"` + }{ + Auths: map[string]struct { + Auth string `json:"auth"` + }{ + host: {Auth: base64.StdEncoding.EncodeToString([]byte(username + ":" + password))}, + }, + } + b, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("encoding registry credential: %v", err) + } + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatalf("writing registry credential to %s: %v", path, err) + } +} + +// isAuthError reports whether the output describes an authentication or +// authorization failure, as opposed to a network, DNS, or naming error that +// would otherwise make the test pass for the wrong reason. +func isAuthError(out string) bool { + out = strings.ToLower(out) + for _, marker := range []string{ + "401", + "403", + "unauthorized", + "denied", + "authentication required", + "invalid username/password", + } { + if strings.Contains(out, marker) { + return true + } + } + return false +} + +// TestOCIRegistryInstallToKubernetes exercises the full flow of pushing a +// chart to an OCI registry and installing it into a real Kubernetes cluster +// using the ambient kubeconfig. It is opt-in because, unlike the registry +// tests, it mutates a cluster. +func TestOCIRegistryInstallToKubernetes(t *testing.T) { + if os.Getenv(envKubernetes) == "" { + t.Skipf("Skipping Kubernetes end-to-end test: set %s to run it against the current kubecontext", envKubernetes) + } + + h := newHarness(t) + h.mustLogin(t) + + // A caller-supplied namespace is left alone; one we create for the run is + // removed again so repeated runs do not accumulate namespaces in the + // cluster. + namespace := os.Getenv(envNamespace) + if namespace == "" { + namespace = "helm-e2e-" + h.runID + t.Cleanup(func() { deleteNamespace(t, namespace) }) + } + repo := h.repo() + + tests := []struct { + name string + chart string + chartName string + chartVersion string + releaseName string + }{ + { + name: "basic chart", + chart: "test-0.1.0.tgz", + chartName: "test", + chartVersion: "0.1.0", + releaseName: "test-release", + }, + { + name: "chart with unicode description", + chart: "unicode-chart-0.1.0.tgz", + chartName: "unicode-chart", + chartVersion: "0.1.0", + releaseName: "unicode-release", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h.mustHelm(t, "push", chart(t, tt.chart), "oci://"+repo) + + // Scope the release name to this run. A caller-supplied namespace + // may already hold releases, and a fixed name could collide with + // one, which cleanup would then uninstall. + releaseName := fmt.Sprintf("%s-%s", tt.releaseName, h.runID) + + t.Cleanup(func() { + if out, err := h.helm(t, "uninstall", releaseName, "--namespace", namespace, "--ignore-not-found", "--wait"); err != nil { + t.Errorf("uninstalling %s failed: %v\n%s", releaseName, err, out) + } + }) + + h.mustHelm(t, "install", releaseName, h.ref(repo, tt.chartName, ""), + "--version", tt.chartVersion, + "--namespace", namespace, + "--create-namespace", + "--wait", + "--timeout", "2m", + ) + + // Read the release back from cluster storage to confirm the + // install really reached the API server. + out := h.mustHelm(t, "status", releaseName, "--namespace", namespace, "--output", "json") + if !strings.Contains(out, `"status":"deployed"`) { + t.Errorf("expected release %s to be deployed, got:\n%s", releaseName, out) + } + }) + } +} + +// assertChartMatches verifies that what was pulled into pullDir is the same +// chart as the fixture at src. Existence and file-vs-directory checks alone +// would pass if the registry served a different artifact under the same tag. +// +// A chart pulled without --untar is the pushed archive verbatim, so it is +// compared byte for byte. With --untar, helm expands the archive, so every +// regular file in the source archive is compared against its extracted +// counterpart. +func assertChartMatches(t *testing.T, src, pullDir string, untarred bool) { + t.Helper() + + if !untarred { + assertFilesEqual(t, src, filepath.Join(pullDir, filepath.Base(src))) + return + } + + want := archiveFiles(t, src) + if len(want) == 0 { + t.Fatalf("fixture %s contains no files", src) + } + for name, content := range want { + extracted := filepath.Join(pullDir, filepath.FromSlash(name)) + got, err := os.ReadFile(extracted) + if err != nil { + t.Errorf("expected extracted file %s: %v", extracted, err) + continue + } + if !bytes.Equal(got, content) { + t.Errorf("extracted file %s does not match the pushed chart", name) + } + } +} + +// assertFilesEqual compares two files byte for byte. +func assertFilesEqual(t *testing.T, want, got string) { + t.Helper() + wantBytes, err := os.ReadFile(want) + if err != nil { + t.Fatalf("reading %s: %v", want, err) + } + gotBytes, err := os.ReadFile(got) + if err != nil { + t.Fatalf("reading %s: %v", got, err) + } + if !bytes.Equal(wantBytes, gotBytes) { + t.Errorf("pulled chart %s does not match the pushed chart %s (%d vs %d bytes)", + got, want, len(gotBytes), len(wantBytes)) + } +} + +// archiveFiles returns the regular files in a gzipped tar archive, keyed by +// their slash-separated path within the archive. +func archiveFiles(t *testing.T, path string) map[string][]byte { + t.Helper() + + f, err := os.Open(path) + if err != nil { + t.Fatalf("opening %s: %v", path, err) + } + defer f.Close() + + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatalf("reading %s as gzip: %v", path, err) + } + defer gz.Close() + + files := map[string][]byte{} + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("reading %s as tar: %v", path, err) + } + if hdr.Typeflag != tar.TypeReg { + continue + } + content, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("reading %s from %s: %v", hdr.Name, path, err) + } + files[hdr.Name] = content + } + return files +} + +// deleteNamespace removes a namespace created by the test run. It is best +// effort in the sense that it reports a failure rather than aborting the test, +// but it does not silently leave the namespace behind. +func deleteNamespace(t *testing.T, namespace string) { + t.Helper() + + // Resolve the cluster exactly the way the helm subprocesses do. They read + // their Kubernetes configuration from the environment (HELM_KUBECONTEXT, + // HELM_KUBEAPISERVER, HELM_KUBETOKEN, HELM_KUBECAFILE and friends), so + // building this client any other way risks installing releases into one + // cluster and sending the namespace delete to another. Such a delete + // returns NotFound, which would look like success while leaking the + // namespace. Going through helm's own settings keeps the two in step + // without having to mirror each variable here. + cfg, err := cli.New().RESTClientGetter().ToRESTConfig() + if err != nil { + t.Errorf("building kube client config to delete namespace %s: %v", namespace, err) + return + } + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + t.Errorf("building kube client to delete namespace %s: %v", namespace, err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + t.Logf("deleting namespace %s", namespace) + if err := client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}); err != nil { + if apierrors.IsNotFound(err) { + return + } + t.Errorf("deleting namespace %s: %v", namespace, err) + return + } + + // Deletion is asynchronous. Wait for the namespace to actually go away so + // that a namespace wedged in Terminating is reported rather than quietly + // accumulating in the cluster. + err = wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { + _, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + }) + if err != nil { + t.Errorf("waiting for namespace %s to be deleted: %v", namespace, err) + } +} diff --git a/test/e2e/testdata/testcharts/compressedchart-0.1.0.tgz b/test/e2e/testdata/testcharts/compressedchart-0.1.0.tgz new file mode 100644 index 000000000..3c9c24d76 Binary files /dev/null and b/test/e2e/testdata/testcharts/compressedchart-0.1.0.tgz differ diff --git a/test/e2e/testdata/testcharts/compressedchart-with-hyphens-0.1.0.tgz b/test/e2e/testdata/testcharts/compressedchart-with-hyphens-0.1.0.tgz new file mode 100644 index 000000000..379210a92 Binary files /dev/null and b/test/e2e/testdata/testcharts/compressedchart-with-hyphens-0.1.0.tgz differ diff --git a/test/e2e/testdata/testcharts/test-0.1.0.tgz b/test/e2e/testdata/testcharts/test-0.1.0.tgz new file mode 100644 index 000000000..9ed772a7f Binary files /dev/null and b/test/e2e/testdata/testcharts/test-0.1.0.tgz differ diff --git a/test/e2e/testdata/testcharts/unicode-chart-0.1.0.tgz b/test/e2e/testdata/testcharts/unicode-chart-0.1.0.tgz new file mode 100644 index 000000000..9af1ce2c8 Binary files /dev/null and b/test/e2e/testdata/testcharts/unicode-chart-0.1.0.tgz differ