From 7a3174af436dc4c0ac33c5bb8a8a5d83b97cfb4b Mon Sep 17 00:00:00 2001 From: Lohit Kolluri Date: Wed, 27 May 2026 11:47:36 +0530 Subject: [PATCH] ci: add checks to prevent invisible Unicode characters in Go source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #32137 Two complementary checks guard against invisible Unicode codepoints that are harmless to the Go compiler but invisible to code reviewers and cause noise in dependency-update tools such as Renovate (surfaced in #32134). 1. bidichk linter (.golangci.yml) Detects bidirectional control characters (U+202A–U+202E, U+2066–U+2069) that underpin "trojan source" attacks (CVE-2021-42574). bidichk has been part of golangci-lint since v1.43.0 and runs inside the existing golangci-lint workflow with no extra dependencies. 2. scripts/check-unicode-invisible.sh + make test-unicode-invisible Detects the ZWSP/BOM class (U+200B–U+200D, U+FEFF) that bidichk does not cover. - find prunes directories by *directory name* (vendor/testdata/third_party/.git) to avoid excluding files that merely contain those substrings. - find stderr is captured and treated as a hard error (exit 2) so traversal problems do not silently produce an incomplete file list. - an empty Go file list is treated as an error (exit 2) to avoid silently passing when run from the wrong directory. - the Python checker reads the NUL-delimited file list from a temp file and fails loudly (exit 2) on UTF-8 decode errors rather than silently replacing invalid bytes. Exit codes are explicit: 0 = clean, 1 = invisible character found, 2 = OS/tool error. Verified against the full source tree (zero false positives) and with synthetic files containing U+200B in spaced paths, under testdata/, and under nested .git/ directories. Signed-off-by: Lohit Kolluri --- .github/workflows/build-test.yml | 2 + .golangci.yml | 1 + Makefile | 9 +++ scripts/check-unicode-invisible.sh | 124 +++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+) create mode 100755 scripts/check-unicode-invisible.sh diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 32c524ed3..12de9b25a 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -28,6 +28,8 @@ jobs: check-latest: true - name: Test source headers are present run: make test-source-headers + - name: Check for invisible Unicode characters (ZWSP/BOM) in Go source + run: make test-unicode-invisible - name: Check if go modules need to be tidied run: go mod tidy -diff - name: Run unit tests diff --git a/.golangci.yml b/.golangci.yml index 1ed3353b4..ca3566f15 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -19,6 +19,7 @@ linters: # Keep sorted alphabetically enable: + - bidichk - depguard - dupl - exhaustive diff --git a/Makefile b/Makefile index 81b149a68..9e5d5aa61 100644 --- a/Makefile +++ b/Makefile @@ -128,6 +128,15 @@ test-style: test-source-headers: @scripts/validate-license.sh +# test-unicode-invisible fails if any Go source file contains invisible Unicode +# codepoints that are invisible to reviewers but cause tooling noise or security +# issues: zero-width spaces (U+200B–U+200D) and byte-order marks (U+FEFF). +# Bidirectional/trojan-source characters (U+202A–U+202E, U+2066–U+2069) are +# covered by the bidichk golangci-lint linter (see .golangci.yml). +.PHONY: test-unicode-invisible +test-unicode-invisible: + @scripts/check-unicode-invisible.sh + .PHONY: test-acceptance test-acceptance: build @if [ -d "${ACCEPTANCE_DIR}" ]; then \ diff --git a/scripts/check-unicode-invisible.sh b/scripts/check-unicode-invisible.sh new file mode 100755 index 000000000..9796f65bb --- /dev/null +++ b/scripts/check-unicode-invisible.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash + +# Copyright The Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# check-unicode-invisible.sh detects invisible Unicode codepoints in Go source +# files that are harmless to a compiler but invisible to reviewers: +# +# U+200B–U+200D zero-width space / non-joiner / joiner (ZWSP class) +# U+FEFF byte-order mark / zero-width no-break space (BOM) +# +# The bidichk golangci-lint linter covers the bidirectional/trojan-source +# range (U+202A–U+202E, U+2066–U+2069). This script fills the gap for the +# ZWSP and BOM codepoints that bidichk does not report. + +set -euo pipefail +IFS=$'\n\t' + +find_go_files() { + # Prune common vendored/generated directories by *directory name* to avoid + # accidentally excluding files that merely contain the substring. + find . \ + \( -type d \( -name .git -o -name testdata -o -name third_party -o -name vendor \) -prune \) \ + -o \( -type f -name '*.go' -print0 \) +} + +FILELIST=$(mktemp "${TMPDIR:-/tmp}/check-unicode-files.XXXXXX") +FINDERR=$(mktemp "${TMPDIR:-/tmp}/check-unicode-finderr.XXXXXX") +trap 'rm -f "$FILELIST" "$FINDERR"' EXIT + +if ! find_go_files >"$FILELIST" 2>"$FINDERR"; then + echo "ERROR: failed to enumerate Go source files." >&2 + cat "$FINDERR" >&2 || true + exit 2 +fi + +# Fail loudly on any traversal errors (permission issues, broken symlinks, etc.) +# rather than silently passing with an incomplete file list. +if [[ -s "$FINDERR" ]]; then + echo "ERROR: errors occurred while enumerating Go source files:" >&2 + cat "$FINDERR" >&2 || true + exit 2 +fi + +# If this script is run from the wrong directory, we may find no Go files. +# Treat that as an error so CI/local runs do not silently pass. +if [[ ! -s "$FILELIST" ]]; then + echo "ERROR: no Go source files found (are you running from the repo root?)" >&2 + exit 2 +fi + +# Run the checker separately from find so enumeration failures are not +# misreported as Unicode detections or Python errors. Exit codes: +# 0 – no invisible characters found +# 1 – one or more invisible characters detected +# 2 – unexpected OS/tool error (file unreadable, etc.) +rc=0 +python3 - "$FILELIST" <<'PYEOF' || rc=$? +import sys + +# Invisible codepoints to detect (ZWSP class + BOM). Bidi/trojan-source chars +# (U+202A-U+202E, U+2066-U+2069) are handled by the bidichk golangci-lint linter. +INVISIBLE = frozenset('\u200b\u200c\u200d\ufeff') + +try: + with open(sys.argv[1], 'rb') as listfh: + paths = listfh.read().split(b'\0') +except OSError as exc: + print('error: could not read file list: {}'.format(exc), file=sys.stderr) + sys.exit(2) + +found = False +for raw in paths: + if not raw: # trailing NUL produces an empty token + continue + path = raw.decode('utf-8', errors='surrogateescape') + try: + with open(path, encoding='utf-8', errors='strict') as fh: + for lineno, text in enumerate(fh, 1): + hits = sorted({c for c in text if c in INVISIBLE}) + if hits: + names = ', '.join('U+{:04X}'.format(ord(c)) for c in hits) + print('{}:{}: invisible Unicode character(s) found: {}'.format( + path, lineno, names)) + found = True + except UnicodeDecodeError as exc: + print('error: could not decode {} as UTF-8: {}'.format(path, exc), file=sys.stderr) + sys.exit(2) + except OSError as exc: + print('error: could not read {}: {}'.format(path, exc), file=sys.stderr) + sys.exit(2) + +sys.exit(1 if found else 0) +PYEOF + +case $rc in + 0) ;; + 1) + echo "FAIL: invisible Unicode character(s) (ZWSP/BOM) detected in Go source." + echo "Remove or replace the offending characters and re-run 'make test-unicode-invisible'." + exit 1 + ;; + 2) + echo "ERROR: unicode check failed while reading Go source files." + echo "Ensure all Go source files are readable and try again." + exit 2 + ;; + *) + echo "ERROR: unicode check failed with unexpected exit code $rc." + echo "Ensure python3 is installed and try again." + exit "$rc" + ;; +esac