From 33e8d8011bfd4617902336b8a4ad9ca430368837 Mon Sep 17 00:00:00 2001 From: Terry Howe Date: Mon, 21 Sep 2026 12:46:22 -0600 Subject: [PATCH] test: run OCI e2e tests across registry implementations The suite has no registry-specific behavior, so covering a new registry is a matter of pointing it at one. Add the CI wiring to do that, and make the repository paths suitable for shared registries. - Repository paths are now stable rather than per-run. The fixtures are immutable, so re-running overwrites each tag with identical content. This bounds the repositories a shared registry accumulates and lets registries that require repositories to exist before a push (ECR) have them created ahead of time. HELM_E2E_ISOLATE restores per-run paths. - Add HELM_E2E_REPO to set the repository path under the registry. - Add e2e.yml: local authenticated registry plus kind, no secrets, so it runs on pull requests from forks. - Add e2e-registries.yml: nightly matrix over GHCR, Quay, and ECR with fail-fast disabled. A registry with no secrets configured is skipped, so they can be wired up one at a time. Signed-off-by: Terry Howe --- .github/workflows/e2e-registries.yml | 143 +++++++++++++++++++++++++++ .github/workflows/e2e.yml | 86 ++++++++++++++++ test/e2e/README.md | 43 +++++++- test/e2e/helper_test.go | 41 +++++++- test/e2e/oci_test.go | 6 +- 5 files changed, 310 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/e2e-registries.yml create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/e2e-registries.yml b/.github/workflows/e2e-registries.yml new file mode 100644 index 000000000..d60058422 --- /dev/null +++ b/.github/workflows/e2e-registries.yml @@ -0,0 +1,143 @@ +name: e2e-registries + +# Runs the OCI end-to-end suite against real registry implementations, to catch +# changes that break a particular registry rather than OCI in general. +# +# This workflow needs credentials, so it deliberately does not run on +# pull_request: fork PRs cannot read secrets and would fail for every outside +# contributor. Secret-free coverage against a local registry runs per-PR in +# e2e.yml. Each registry is a separate matrix leg with fail-fast disabled, so +# one registry being down does not hide the results of the others. +# +# Required secrets, per registry. A leg whose secrets are absent is skipped +# rather than failed, so registries can be wired up one at a time. +# +# GHCR HELM_E2E_GHCR_REGISTRY e.g. ghcr.io/helm +# HELM_E2E_GHCR_USERNAME +# HELM_E2E_GHCR_TOKEN PAT with write:packages +# +# Quay HELM_E2E_QUAY_REGISTRY e.g. quay.io/helm +# HELM_E2E_QUAY_USERNAME robot account, e.g. helm+e2e +# HELM_E2E_QUAY_TOKEN robot account token +# +# ECR HELM_E2E_ECR_REGISTRY e.g. 111122223333.dkr.ecr.us-east-1.amazonaws.com +# HELM_E2E_ECR_REGION e.g. us-east-1 +# HELM_E2E_ECR_ACCESS_KEY_ID +# HELM_E2E_ECR_SECRET_ACCESS_KEY +# +# ECR is the odd one out in two ways: it mints a short-lived password rather +# than using a static one, and it does not create repositories on push, so the +# repositories are created below before the tests run. + +on: + schedule: + # Nightly, after most merges have landed. + - cron: "0 6 * * *" + workflow_dispatch: + +permissions: + contents: read + +env: + KIND_VERSION: v0.31.0 + # Repository path under each registry. Stable rather than per-run, so the + # set of repositories stays bounded and can be pre-created for ECR. + HELM_E2E_REPO: helm-e2e + +jobs: + registries: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + name: [ghcr, quay, ecr] + name: ${{ matrix.name }} + steps: + # Secrets cannot be referenced from strategy.matrix, so every registry's + # secrets are bound here and the matrix leg selects among them. They are + # read as environment variables rather than interpolated into the script, + # so no credential becomes shell source. + - name: Resolve registry configuration + id: config + env: + MATRIX_NAME: ${{ matrix.name }} + GHCR_REGISTRY: ${{ secrets.HELM_E2E_GHCR_REGISTRY }} + GHCR_USERNAME: ${{ secrets.HELM_E2E_GHCR_USERNAME }} + GHCR_PASSWORD: ${{ secrets.HELM_E2E_GHCR_TOKEN }} + QUAY_REGISTRY: ${{ secrets.HELM_E2E_QUAY_REGISTRY }} + QUAY_USERNAME: ${{ secrets.HELM_E2E_QUAY_USERNAME }} + QUAY_PASSWORD: ${{ secrets.HELM_E2E_QUAY_TOKEN }} + ECR_REGISTRY: ${{ secrets.HELM_E2E_ECR_REGISTRY }} + # ECR authenticates as the fixed user "AWS" with a token minted below. + ECR_USERNAME: AWS + ECR_PASSWORD: "" + run: | + set -euo pipefail + prefix="$(printf '%s' "${MATRIX_NAME}" | tr '[:lower:]' '[:upper:]')" + registry="${prefix}_REGISTRY" + username="${prefix}_USERNAME" + password="${prefix}_PASSWORD" + + # A registry with no secrets set is skipped rather than failed, so + # registries can be wired up one at a time. + if [ -z "${!registry:-}" ]; then + echo "no registry secret set for ${MATRIX_NAME}, skipping this leg" + echo "configured=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + { + echo "HELM_E2E_REGISTRY=${!registry}" + echo "HELM_E2E_USERNAME=${!username:-}" + } >> "$GITHUB_ENV" + if [ -n "${!password:-}" ]; then + echo "HELM_E2E_PASSWORD=${!password}" >> "$GITHUB_ENV" + fi + echo "configured=true" >> "$GITHUB_OUTPUT" + + - name: Checkout source code + if: steps.config.outputs.configured == 'true' + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # pin@v6.0.0 + - name: Add variables to environment file + if: steps.config.outputs.configured == 'true' + run: cat ".github/env" >> "$GITHUB_ENV" + - name: Setup Go + if: steps.config.outputs.configured == 'true' + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # pin@6.1.0 + with: + go-version: '${{ env.GOLANG_VERSION }}' + check-latest: true + + # ECR issues a short-lived authorization token rather than accepting a + # long-lived secret, and does not create repositories on push. + - name: Prepare ECR + if: steps.config.outputs.configured == 'true' && matrix.name == 'ecr' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.HELM_E2E_ECR_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.HELM_E2E_ECR_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.HELM_E2E_ECR_REGION }} + run: | + set -euo pipefail + # Helm appends the chart name to HELM_E2E_REPO, so each fixture needs + # its own repository. Keep this list in step with + # test/e2e/testdata/testcharts. + for chart in test compressedchart compressedchart-with-hyphens unicode-chart; do + aws ecr describe-repositories --repository-names "${HELM_E2E_REPO}/${chart}" >/dev/null 2>&1 \ + || aws ecr create-repository --repository-name "${HELM_E2E_REPO}/${chart}" >/dev/null + done + token="$(aws ecr get-login-password)" + echo "::add-mask::${token}" + echo "HELM_E2E_PASSWORD=${token}" >> "$GITHUB_ENV" + + - name: Create a kind cluster + if: steps.config.outputs.configured == 'true' + run: | + set -euo pipefail + go install "sigs.k8s.io/kind@${KIND_VERSION}" + kind create cluster --name helm-e2e --wait 120s + + - name: Run end-to-end tests + if: steps.config.outputs.configured == 'true' + env: + HELM_E2E_KUBERNETES: "true" + run: make test-e2e diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..da64696fe --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,86 @@ +name: e2e + +on: + push: + branches: + - "main" + - "dev-v3" + - "release-**" + pull_request: + branches: + - "main" + - "dev-v3" + workflow_dispatch: + +permissions: + contents: read + +env: + KIND_VERSION: v0.31.0 + REGISTRY_IMAGE: registry:2.8.3 + HTPASSWD_IMAGE: httpd:2.4-alpine + +jobs: + # Runs the OCI end-to-end suite against a local, authenticated registry and a + # kind cluster. Needs no secrets, so it runs on pull requests from forks. + # Cross-registry coverage lives in e2e-registries.yml. + local-registry: + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # pin@v6.0.0 + - name: Add variables to environment file + run: cat ".github/env" >> "$GITHUB_ENV" + - name: Setup Go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # pin@6.1.0 + with: + go-version: '${{ env.GOLANG_VERSION }}' + check-latest: true + + - name: Start an authenticated local registry + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/registry-auth" + docker run --rm --entrypoint htpasswd "${HTPASSWD_IMAGE}" \ + -Bbn e2euser e2epass > "${RUNNER_TEMP}/registry-auth/htpasswd" + docker run -d --name helm-e2e-registry -p 5000:5000 \ + -v "${RUNNER_TEMP}/registry-auth:/auth" \ + -e REGISTRY_AUTH=htpasswd \ + -e REGISTRY_AUTH_HTPASSWD_REALM=Registry \ + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ + "${REGISTRY_IMAGE}" + # Wait for the registry to answer before handing it to the tests. An + # unauthenticated /v2/ must be rejected, which also confirms that the + # htpasswd file was picked up and the invalid-credential test will + # have something to assert against. + for _ in $(seq 1 30); do + if [ "$(curl -s -o /dev/null -w '%{http_code}' http://localhost:5000/v2/)" = "401" ]; then + echo "registry is up and requires authentication" + exit 0 + fi + sleep 1 + done + echo "local registry did not come up with authentication enabled" + docker logs helm-e2e-registry + exit 1 + + - name: Create a kind cluster + run: | + set -euo pipefail + go install "sigs.k8s.io/kind@${KIND_VERSION}" + kind create cluster --name helm-e2e --wait 120s + kubectl cluster-info --context kind-helm-e2e + + - name: Run end-to-end tests + env: + HELM_E2E_REGISTRY: localhost:5000 + HELM_E2E_REPO: helm-e2e + HELM_E2E_USERNAME: e2euser + HELM_E2E_PASSWORD: e2epass + HELM_E2E_PLAIN_HTTP: "true" + HELM_E2E_KUBERNETES: "true" + run: make test-e2e + + - name: Collect registry logs on failure + if: failure() + run: docker logs helm-e2e-registry diff --git a/test/e2e/README.md b/test/e2e/README.md index 678cec038..ba3c3fd7d 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -36,6 +36,8 @@ auth-enabled registry to cover that path. | `HELM_E2E_REGISTRY` | yes | Registry namespace to push to, without a scheme (e.g. `ghcr.io/your-org`). | | `HELM_E2E_USERNAME` | yes | Username for the registry. | | `HELM_E2E_PASSWORD` | yes | Password or token for the registry. Passed to helm on stdin and never logged. | +| `HELM_E2E_REPO` | no | Repository path under the registry. Defaults to `helm-e2e`. | +| `HELM_E2E_ISOLATE` | no | Append a per-run suffix to the repository path. Off by default; see below. | | `HELM_E2E_BIN` | no | Path to a prebuilt helm binary. Defaults to building one from the working tree. | | `HELM_E2E_PLAIN_HTTP` | no | Set to use plain HTTP, for registries without TLS. | | `HELM_E2E_KUBERNETES` | no | Set to run the install test against the current kubecontext. Off by default because it mutates. | @@ -44,8 +46,45 @@ auth-enabled registry to cover that path. Because the `e2e` build tag is already an explicit opt-in, a missing required variable fails the test rather than silently skipping it. -Each run pushes to repositories suffixed with a random run ID, so concurrent -runs and repeated runs in a shared namespace do not collide. +## Repository layout + +Helm appends the chart name to the push target, so a run touches one repository +per fixture: + +``` +//test +//compressedchart +//compressedchart-with-hyphens +//unicode-chart +``` + +That set is stable by default. The fixtures are immutable, so re-running +overwrites each tag with byte-identical content: concurrent runs do not +interfere, a shared registry does not accumulate repositories over time, and +registries that require a repository to exist before a push (ECR) can have +these created ahead of time. + +Set `HELM_E2E_ISOLATE` to append a per-run suffix instead, which is useful when +several people share one registry namespace and you want a run to stand alone. +Note that stable paths stay safe only while every test pushes identical content +under a given tag; a test that pushes mutated content under a fixed tag would +need isolation. + +## Running against several registries + +The suite has no registry-specific behavior — everything comes from the +environment — so covering a new registry is a matter of pointing the same tests +at it. This is how `.github/workflows/e2e-registries.yml` exercises GHCR, Quay, +and ECR on a schedule, one matrix leg each, to catch changes that break a +particular registry implementation rather than OCI in general. + +Two registry differences are worth knowing about: + +- A default `registry:2` serves anonymous access, so the invalid-credential + test skips itself there, as described above. +- ECR does not create repositories on push and issues a short-lived token + rather than accepting a static password. The workflow creates the four + repositories listed above and mints a token before running the tests. The Kubernetes test deletes the namespace it creates. If you supply `HELM_E2E_NAMESPACE`, that namespace is left in place and only the releases are diff --git a/test/e2e/helper_test.go b/test/e2e/helper_test.go index 91ed186ac..7d0c1cf84 100644 --- a/test/e2e/helper_test.go +++ b/test/e2e/helper_test.go @@ -51,6 +51,15 @@ const ( // envPlainHTTP requests plain HTTP for registries without TLS, which is // useful when testing against a local registry. envPlainHTTP = "HELM_E2E_PLAIN_HTTP" + // envRepo is the repository path under the registry that charts are + // pushed to. Helm appends the chart name, so this is a prefix shared by + // every fixture. + envRepo = "HELM_E2E_REPO" + // envIsolate appends a per-run suffix to the repository path. It is off by + // default so that the set of repositories a run touches is stable and can + // be created ahead of time on registries that do not create repositories + // on push. + envIsolate = "HELM_E2E_ISOLATE" // envKubernetes opts in to the tests that install charts into a real // Kubernetes cluster using the ambient kubeconfig. envKubernetes = "HELM_E2E_KUBERNETES" @@ -70,6 +79,10 @@ type harness struct { password string // plainHTTP is true when the registry should be reached over HTTP. plainHTTP bool + // repoBase is the repository path charts are pushed under. + repoBase string + // isolate appends runID to repoBase. + isolate bool // configPath is the registry credential file used for this run. configPath string // runID isolates the repositories written by a single test run so that @@ -88,6 +101,8 @@ func newHarness(t *testing.T) *harness { username: requireEnv(t, envUsername), password: requireEnv(t, envPassword), plainHTTP: os.Getenv(envPlainHTTP) != "", + repoBase: envOr(envRepo, "helm-e2e"), + isolate: os.Getenv(envIsolate) != "", helmBin: helmBinary(t), runID: runID(t), } @@ -146,10 +161,28 @@ func (h *harness) registryHost() string { return host } -// repo returns an isolated repository path for this run, so that a shared -// registry namespace can serve many runs without interference. -func (h *harness) repo(name string) string { - return fmt.Sprintf("%s/%s-%s", h.registry, name, h.runID) +// repo returns the repository path charts are pushed to. Helm appends the +// chart name, so a run touches one repository per fixture. +// +// The path is stable by default. Re-pushing a fixture overwrites the same tag +// with byte-identical content, so concurrent runs do not interfere, the set of +// repositories stays bounded, and registries that require repositories to +// exist before a push (ECR) can have them created ahead of time. Set +// HELM_E2E_ISOLATE to give a run its own repositories instead. +func (h *harness) repo() string { + if h.isolate { + return fmt.Sprintf("%s/%s-%s", h.registry, h.repoBase, h.runID) + } + return fmt.Sprintf("%s/%s", h.registry, h.repoBase) +} + +// envOr returns the value of the named environment variable, or def when it is +// unset. +func envOr(name, def string) string { + if v := os.Getenv(name); v != "" { + return v + } + return def } // ref returns a full OCI reference for a chart within an isolated repository. diff --git a/test/e2e/oci_test.go b/test/e2e/oci_test.go index 92699fcbe..076419cda 100644 --- a/test/e2e/oci_test.go +++ b/test/e2e/oci_test.go @@ -45,7 +45,7 @@ func TestOCIRegistryPushPull(t *testing.T) { h := newHarness(t) h.mustLogin(t) - repo := h.repo("push-pull") + repo := h.repo() contentCache := t.TempDir() tests := []struct { @@ -142,7 +142,7 @@ func TestOCIRegistryInvalidCredentials(t *testing.T) { // locally for want of any credential at all. writeRegistryCredential(t, h.registryConfig(t), h.registryHost(), h.username, "invalid-"+h.runID) - out, err = h.helm(t, "push", chart(t, "test-0.1.0.tgz"), "oci://"+h.repo("auth-failure")) + out, err = h.helm(t, "push", chart(t, "test-0.1.0.tgz"), "oci://"+h.repo()) if err == nil { t.Fatal("expected push to fail with an invalid credential, but it succeeded") } @@ -216,7 +216,7 @@ func TestOCIRegistryInstallToKubernetes(t *testing.T) { namespace = "helm-e2e-" + h.runID t.Cleanup(func() { deleteNamespace(t, namespace) }) } - repo := h.repo("install") + repo := h.repo() tests := []struct { name string