From 00e7d7b6896bfd89948e266705f223645d167d8f Mon Sep 17 00:00:00 2001 From: Hsiu-Chi Tsai <84045975+thc1006@users.noreply.github.com> Date: Fri, 25 Jul 2025 01:12:54 +0800 Subject: [PATCH] Ensure coverage script uses TESTFLAGS for consistent test execution behavior with shuffle and other flags across all test modes. Fixes issue #30892. Signed-off-by: Hsiu-Chi Tsai <84045975+thc1006@users.noreply.github.com> --- .golangci.yml | 83 ++++++++++++++------------------------------- CONTRIBUTING.md | 62 ++++++++++++++++++++++++++++----- scripts/coverage.sh | 2 +- 3 files changed, 80 insertions(+), 67 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index a9b13c35f..03ddf16cd 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,22 +1,8 @@ -formatters: - enable: - - gofmt - - goimports - - exclusions: - generated: lax - - settings: - gofmt: - simplify: true - - goimports: - local-prefixes: - - helm.sh/helm/v4 +run: + timeout: 10m linters: - default: none - + disable-all: true enable: - depguard - dupl @@ -27,45 +13,28 @@ linters: - nakedret - revive - staticcheck - - thelper - unused - - usestdlibvars - - usetesting - - exclusions: - generated: lax - - presets: - - comments - - common-false-positives - - legacy - - std-error-handling - - rules: [] - - warn-unused: true - - settings: - depguard: - rules: - Main: - deny: - - pkg: github.com/hashicorp/go-multierror - desc: "use errors instead" - - pkg: github.com/pkg/errors - desc: "use errors instead" - - dupl: - threshold: 400 - - gomodguard: - blocked: - modules: - - github.com/evanphx/json-patch: - recommendations: - - github.com/evanphx/json-patch/v5 - -run: - timeout: 10m + - gofmt + - goimports -version: "2" +linters-settings: + gofmt: + simplify: true + goimports: + local-prefixes: helm.sh/helm/v4 + depguard: + rules: + main: + deny: + - pkg: github.com/hashicorp/go-multierror + desc: "use errors instead" + - pkg: github.com/pkg/errors + desc: "use errors instead" + dupl: + threshold: 400 + gomodguard: + blocked: + modules: + - github.com/evanphx/json-patch: + recommendations: + - github.com/evanphx/json-patch/v5 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3897a64c1..e27957ab2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -232,25 +232,69 @@ docs](https://helm.sh/docs/developers/). To run the test suite, use the `make test` command. This will run all the unit tests. -By default, the tests are run in a randomized order and each test is run 3 times. This is to help -detect flaky tests. If you want to run the tests in a specific order, you can use the `-shuffle=off` -flag. If you want to run the tests a different number of times, you can use the `-count=n` flag, -where `n` is the number of times you want to run the tests. +#### Test Configuration Variables -For example, to run the tests in order and only once, you can use the following command: +Helm uses two main variables to configure test execution: +- **TESTFLAGS**: Controls test execution flags (default: `-shuffle=on`) +- **TESTCOUNT**: Controls how many times tests are run (default: 1, CI uses 2) + +#### Default Test Behavior + +By default, tests are run in a randomized order with shuffle enabled. This helps detect order-dependent and flaky tests that might pass in development but fail in different environments. + +#### Common Test Commands + +Run tests with default settings (shuffle enabled, run once): ```bash -make test TESTFLAGS="-shuffle=off -count=1" +make test ``` -If a test fails, the seed that was used to randomize the tests will be printed. You can use this -seed to reproduce the failure. For example, if the seed was `12345`, you can use the following -command to reproduce the failure: +Run tests without shuffle for faster debugging: +```bash +make test TESTFLAGS="-shuffle=off" +``` +Run tests multiple times to catch flaky behavior: +```bash +make test TESTCOUNT=3 +``` + +Run tests with a specific seed to reproduce failures: ```bash make test TESTFLAGS="-shuffle=on -seed=12345" ``` +Run tests multiple times with a specific seed: +```bash +make test TESTFLAGS="-shuffle=on -seed=12345" TESTCOUNT=3 +``` + +Run only coverage tests: +```bash +make test-coverage +``` + +#### CI Behavior + +The continuous integration system runs tests with enhanced settings to catch more issues: +- Tests are run twice (`TESTCOUNT=2`) to detect flaky behavior +- Shuffle is always enabled with random seeds +- Both unit tests and coverage tests are executed + +#### Reproducing Test Failures + +If a test fails, the seed used for randomization will be printed in the output. Use this seed to reproduce the exact failure: + +```bash +make test TESTFLAGS="-shuffle=on -seed=1642529943" +``` + +For persistent issues, run the test multiple times with the same seed: +```bash +make test TESTFLAGS="-shuffle=on -seed=1642529943" TESTCOUNT=5 +``` + ## Pull Requests Like any good open source project, we use Pull Requests (PRs) to track code changes. diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 034dca830..2e56b2557 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -24,7 +24,7 @@ generate_cover_data() { for d in $(go list ./...) ; do ( local output="${coverdir}/${d//\//-}.cover" - go test -coverprofile="${output}" -covermode="$covermode" "$d" + go test -coverprofile="${output}" -covermode="$covermode" ${TESTFLAGS:-} "$d" ) done