From 5d90868af0fbb96d7694dbe0102bc524542858fc Mon Sep 17 00:00:00 2001 From: Gianni Date: Thu, 20 Nov 2025 13:22:31 +0100 Subject: [PATCH] Feature/add git based dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This enhancement adds support for using Git repositories as chart dependency sources. Charts can be fetched directly from Git repositories during `helm dep build` and `helm dep update` operations. Example: ```yaml dependencies: - name: mychart version: "v1.2.3" # ← This is the Git ref (tag/branch/commit) repository: git+https://github.com/user/repo?path=charts/mychart ``` Supported URL Formats: - `git://` - `git+https://` - `git+http://` - `git+ssh://` --------- Signed-off-by: Gianni Carafa --- internal/resolver/resolver.go | 41 ++++- pkg/downloader/chart_downloader.go | 47 ++++- pkg/downloader/manager.go | 45 ++++- pkg/getter/getter.go | 8 + pkg/getter/getter_test.go | 4 +- pkg/getter/gitgetter.go | 277 +++++++++++++++++++++++++++++ pkg/getter/gitgetter_test.go | 142 +++++++++++++++ 7 files changed, 551 insertions(+), 13 deletions(-) create mode 100644 pkg/getter/gitgetter.go create mode 100644 pkg/getter/gitgetter_test.go diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 3efe94f10..e05654677 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -59,11 +59,6 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string locked := make([]*chart.Dependency, len(reqs)) missing := []string{} for i, d := range reqs { - constraint, err := semver.NewConstraint(d.Version) - if err != nil { - return nil, fmt.Errorf("dependency %q has an invalid version/constraint format: %w", d.Name, err) - } - if d.Repository == "" { // Local chart subfolder if _, err := GetLocalPath(filepath.Join("charts", d.Name), r.chartpath); err != nil { @@ -77,6 +72,24 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string } continue } + + // Handle Git repositories - they don't use semver constraints or index files + // The version field is used as-is as the Git ref (branch/tag/commit) + if isGitRepository(d.Repository) { + locked[i] = &chart.Dependency{ + Name: d.Name, + Repository: d.Repository, + Version: d.Version, + } + continue + } + + // For non-Git repositories, parse version as semver constraint + constraint, err := semver.NewConstraint(d.Version) + if err != nil { + return nil, fmt.Errorf("dependency %q has an invalid version/constraint format: %w", d.Name, err) + } + if strings.HasPrefix(d.Repository, "file://") { chartpath, err := GetLocalPath(d.Repository, r.chartpath) if err != nil { @@ -107,6 +120,16 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string continue } + // Handle Git repositories - they don't use index files + if isGitRepository(d.Repository) { + locked[i] = &chart.Dependency{ + Name: d.Name, + Repository: d.Repository, + Version: d.Version, + } + continue + } + repoName := repoNames[d.Name] // if the repository was not defined, but the dependency defines a repository url, bypass the cache if repoName == "" && d.Repository != "" { @@ -261,3 +284,11 @@ func GetLocalPath(repo, chartpath string) (string, error) { return depPath, nil } + +// isGitRepository checks if a repository URL is a Git repository +func isGitRepository(repo string) bool { + return strings.HasPrefix(repo, "git://") || + strings.HasPrefix(repo, "git+https://") || + strings.HasPrefix(repo, "git+http://") || + strings.HasPrefix(repo, "git+ssh://") +} diff --git a/pkg/downloader/chart_downloader.go b/pkg/downloader/chart_downloader.go index 00c8c56e8..098294c9a 100644 --- a/pkg/downloader/chart_downloader.go +++ b/pkg/downloader/chart_downloader.go @@ -32,6 +32,7 @@ import ( "helm.sh/helm/v4/internal/fileutil" ifs "helm.sh/helm/v4/internal/third_party/dep/fs" "helm.sh/helm/v4/internal/urlutil" + "helm.sh/helm/v4/pkg/chart/v2/loader" "helm.sh/helm/v4/pkg/getter" "helm.sh/helm/v4/pkg/helmpath" "helm.sh/helm/v4/pkg/provenance" @@ -153,6 +154,16 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven if u.Scheme == registry.OCIScheme { idx := strings.LastIndexByte(name, ':') name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) + } else if isGitURL(u.Scheme) { + // For Git URLs, extract the chart name and version from the tarball + // to generate the proper filename + chartName, chartVersion, err := extractChartMetadata(data.Bytes()) + if err == nil && chartName != "" && chartVersion != "" { + name = fmt.Sprintf("%s-%s.tgz", chartName, chartVersion) + } else { + // Fallback: use .tgz extension if extraction fails + name = filepath.Base(u.Path) + ".tgz" + } } destfile := filepath.Join(dest, name) @@ -227,13 +238,10 @@ func (c *ChartDownloader) DownloadToCache(ref, version string) (string, *provena // Check the cache for the file digest, err := hex.DecodeString(digestString) if err != nil { - return "", nil, err + return "", nil, fmt.Errorf("unable to decode digest: %w", err) } var digest32 [32]byte copy(digest32[:], digest) - if err != nil { - return "", nil, fmt.Errorf("unable to decode digest: %w", err) - } var pth string // only fetch from the cache if we have a digest @@ -367,6 +375,17 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (string, *url return digest, OCIref, err } + // Handle Git repositories + if isGitURL(u.Scheme) { + // Git URLs are handled directly by the Git getter + // Pass the version to the getter so it can use it as the Git ref + c.Options = append(c.Options, getter.WithURL(ref)) + if version != "" { + c.Options = append(c.Options, getter.WithTagName(version)) + } + return "", u, nil + } + rf, err := loadRepoConfig(c.RepositoryConfig) if err != nil { return "", u, err @@ -581,3 +600,23 @@ func loadRepoConfig(file string) (*repo.File, error) { } return r, nil } + +// isGitURL checks if the scheme is a Git URL scheme +func isGitURL(scheme string) bool { + return scheme == "git" || scheme == "git+https" || scheme == "git+http" || scheme == "git+ssh" +} + +// extractChartMetadata extracts the chart name and version from a tarball +func extractChartMetadata(data []byte) (string, string, error) { + // Load the chart from the tarball data + ch, err := loader.LoadArchive(bytes.NewReader(data)) + if err != nil { + return "", "", err + } + + if ch.Metadata == nil { + return "", "", fmt.Errorf("chart metadata is nil") + } + + return ch.Metadata.Name, ch.Metadata.Version, nil +} diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index 6043fbaaa..6344ae60d 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -313,6 +313,30 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { continue } + // Handle Git repositories + if isGitRepository(dep.Repository) { + if m.Debug { + fmt.Fprintf(m.Out, "Downloading %s from git repo %s\n", dep.Name, dep.Repository) + } + + dl := ChartDownloader{ + Out: m.Out, + Verify: m.Verify, + Keyring: m.Keyring, + RepositoryConfig: m.RepositoryConfig, + RepositoryCache: m.RepositoryCache, + ContentCache: m.ContentCache, + RegistryClient: m.RegistryClient, + Getters: m.Getters, + } + + if _, _, err = dl.DownloadTo(dep.Repository, dep.Version, tmpPath); err != nil { + saveError = fmt.Errorf("could not download %s from git: %w", dep.Repository, err) + break + } + continue + } + // Any failure to resolve/download a chart should fail: // https://github.com/helm/helm/issues/1439 churl, username, password, insecureSkipTLSVerify, passCredentialsAll, caFile, certFile, keyFile, err := m.findChartURL(dep.Name, dep.Version, dep.Repository, repos) @@ -475,8 +499,8 @@ func (m *Manager) hasAllRepos(deps []*chart.Dependency) error { missing := []string{} Loop: for _, dd := range deps { - // If repo is from local path or OCI, continue - if strings.HasPrefix(dd.Repository, "file://") || registry.IsOCI(dd.Repository) { + // If repo is from local path, OCI, or Git, continue + if strings.HasPrefix(dd.Repository, "file://") || registry.IsOCI(dd.Repository) || isGitRepository(dd.Repository) { continue } @@ -598,6 +622,15 @@ func (m *Manager) resolveRepoNames(deps []*chart.Dependency) (map[string]string, continue } + // if dep chart is from git repository + if isGitRepository(dd.Repository) { + if m.Debug { + fmt.Fprintf(m.Out, "Repository from git: %s\n", dd.Repository) + } + reposMap[dd.Name] = dd.Repository + continue + } + found := false for _, repo := range repos { @@ -920,3 +953,11 @@ func key(name string) (string, error) { } return hex.EncodeToString(hash.Sum(nil)), nil } + +// isGitRepository checks if a repository URL is a Git repository +func isGitRepository(repo string) bool { + return strings.HasPrefix(repo, "git://") || + strings.HasPrefix(repo, "git+https://") || + strings.HasPrefix(repo, "git+http://") || + strings.HasPrefix(repo, "git+ssh://") +} diff --git a/pkg/getter/getter.go b/pkg/getter/getter.go index a2d0f0ee2..a27221627 100644 --- a/pkg/getter/getter.go +++ b/pkg/getter/getter.go @@ -218,6 +218,14 @@ func Getters(extraOpts ...Option) Providers { return NewOCIGetter(options...) }, }, + Provider{ + Schemes: []string{"git", "git+https", "git+http", "git+ssh"}, + New: func(options ...Option) (Getter, error) { + options = append(options, defaultOptions...) + options = append(options, extraOpts...) + return NewGitGetter(options...) + }, + }, } } diff --git a/pkg/getter/getter_test.go b/pkg/getter/getter_test.go index 83920e809..a8750c174 100644 --- a/pkg/getter/getter_test.go +++ b/pkg/getter/getter_test.go @@ -75,8 +75,8 @@ func TestAll(t *testing.T) { env.PluginsDirectory = pluginDir all := All(env) - if len(all) != 4 { - t.Errorf("expected 4 providers (default plus three plugins), got %d", len(all)) + if len(all) != 5 { + t.Errorf("expected 5 providers (3 default plus three plugins), got %d", len(all)) } if _, err := all.ByScheme("test2"); err != nil { diff --git a/pkg/getter/gitgetter.go b/pkg/getter/gitgetter.go new file mode 100644 index 000000000..559045547 --- /dev/null +++ b/pkg/getter/gitgetter.go @@ -0,0 +1,277 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package getter + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "log/slog" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// GitGetter handles fetching charts from Git repositories +type GitGetter struct { + opts getterOptions +} + +// Get performs a Get from a Git repository and returns the body. +func (g *GitGetter) Get(href string, options ...Option) (*bytes.Buffer, error) { + for _, opt := range options { + opt(&g.opts) + } + return g.get(href) +} + +// get clones a Git repository, packages the chart, and returns it as a buffer +func (g *GitGetter) get(href string) (*bytes.Buffer, error) { + // Parse the Git URL + // Format: git://github.com/user/repo@ref?path=charts/mychart + // Or: git+https://github.com/user/repo@ref?path=charts/mychart + + repoURL, ref, chartPath, err := parseGitURL(href) + if err != nil { + return nil, fmt.Errorf("failed to parse git URL: %w", err) + } + + // Use version from options if provided (takes precedence over URL ref) + // This allows the dependency version field to specify the Git ref + if g.opts.version != "" && g.opts.version != "*" { + ref = g.opts.version + } + + // Create a temporary directory for cloning + tmpDir, err := os.MkdirTemp("", "helm-git-") + if err != nil { + return nil, fmt.Errorf("failed to create temp directory: %w", err) + } + defer os.RemoveAll(tmpDir) + + slog.Debug("cloning git repository", "url", repoURL, "ref", ref, "path", chartPath) + + // Clone the repository + if err := g.cloneRepo(repoURL, ref, tmpDir); err != nil { + return nil, fmt.Errorf("failed to clone repository: %w", err) + } + + // Determine the chart directory + chartDir := tmpDir + if chartPath != "" { + chartDir = filepath.Join(tmpDir, chartPath) + } + + // Check if the chart directory exists + if _, err := os.Stat(chartDir); os.IsNotExist(err) { + return nil, fmt.Errorf("chart path %q does not exist in repository", chartPath) + } + + // Package the chart into a tarball + tarData, err := g.packageChart(chartDir) + if err != nil { + return nil, fmt.Errorf("failed to package chart: %w", err) + } + + return tarData, nil +} + +// cloneRepo clones a Git repository to the specified directory +func (g *GitGetter) cloneRepo(repoURL, ref, destDir string) error { + // Use shallow clone for better performance + args := []string{"clone", "--depth", "1"} + + // If a specific ref is provided, clone that branch/tag + if ref != "" && ref != "HEAD" && ref != "master" && ref != "main" { + args = append(args, "--branch", ref) + } + + args = append(args, repoURL, destDir) + + cmd := exec.Command("git", args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + // If shallow clone with branch failed, try full clone and checkout + if ref != "" && ref != "HEAD" && ref != "master" && ref != "main" { + slog.Debug("shallow clone failed, trying full clone", "error", stderr.String()) + return g.fullCloneAndCheckout(repoURL, ref, destDir) + } + return fmt.Errorf("git clone failed: %s", stderr.String()) + } + + // If ref is specified but wasn't used in clone (HEAD, master, main), checkout now + if ref != "" && (ref == "HEAD" || ref == "master" || ref == "main") { + cmd := exec.Command("git", "-C", destDir, "checkout", ref) + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("git checkout failed: %s", stderr.String()) + } + } + + return nil +} + +// fullCloneAndCheckout performs a full clone and checks out a specific ref (commit SHA, tag, or branch) +func (g *GitGetter) fullCloneAndCheckout(repoURL, ref, destDir string) error { + var stderr bytes.Buffer + + // Remove the directory if it exists from failed shallow clone + os.RemoveAll(destDir) + + // Full clone + cmd := exec.Command("git", "clone", repoURL, destDir) + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("git clone failed: %s", stderr.String()) + } + + // Checkout the specific ref + cmd = exec.Command("git", "-C", destDir, "checkout", ref) + stderr.Reset() + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("git checkout failed for ref %q: %s", ref, stderr.String()) + } + + return nil +} + +// packageChart packages a chart directory into a tarball +func (g *GitGetter) packageChart(chartDir string) (*bytes.Buffer, error) { + // Use helm package command to create the tarball + tmpDir, err := os.MkdirTemp("", "helm-git-package-") + if err != nil { + return nil, fmt.Errorf("failed to create temp directory for packaging: %w", err) + } + defer os.RemoveAll(tmpDir) + + // Use --dependency-update to automatically fetch any dependencies the chart needs + // This handles charts that have their own dependencies + cmd := exec.Command("helm", "package", chartDir, "-d", tmpDir, "--dependency-update") + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("helm package failed: %s", stderr.String()) + } + + // Find the created tarball + entries, err := os.ReadDir(tmpDir) + if err != nil { + return nil, fmt.Errorf("failed to read package directory: %w", err) + } + + if len(entries) == 0 { + return nil, fmt.Errorf("no package file created") + } + + // Read the tarball + tarPath := filepath.Join(tmpDir, entries[0].Name()) + tarData, err := os.ReadFile(tarPath) + if err != nil { + return nil, fmt.Errorf("failed to read package file: %w", err) + } + + return bytes.NewBuffer(tarData), nil +} + +// parseGitURL parses a Git URL and extracts the repository URL, ref, and chart path +// Supported formats: +// - git://github.com/user/repo@ref?path=charts/mychart +// - git+https://github.com/user/repo@ref?path=charts/mychart +// - git+ssh://git@github.com/user/repo@ref?path=charts/mychart +func parseGitURL(href string) (repoURL, ref, chartPath string, err error) { + u, err := url.Parse(href) + if err != nil { + return "", "", "", err + } + + // Extract the ref from the URL fragment or path + ref = "HEAD" // default ref + repoPath := u.Path + + // Check if ref is specified with @ symbol + if strings.Contains(u.Path, "@") { + parts := strings.SplitN(u.Path, "@", 2) + repoPath = parts[0] + ref = parts[1] + } + + // Check if ref is specified as a query parameter + if u.Query().Get("ref") != "" { + ref = u.Query().Get("ref") + } + + // Extract chart path from query parameter + chartPath = u.Query().Get("path") + + // Reconstruct the repository URL + scheme := u.Scheme + if strings.HasPrefix(scheme, "git+") { + scheme = strings.TrimPrefix(scheme, "git+") + } else if scheme == "git" { + // Convert git:// to https:// for better compatibility + scheme = "https" + } + + // Handle git+ssh special case + if scheme == "ssh" { + // git+ssh://git@github.com/user/repo becomes git@github.com:user/repo + if u.User != nil { + username := u.User.Username() + repoURL = fmt.Sprintf("%s@%s:%s", username, u.Host, strings.TrimPrefix(repoPath, "/")) + } else { + repoURL = fmt.Sprintf("git@%s:%s", u.Host, strings.TrimPrefix(repoPath, "/")) + } + } else { + repoURL = fmt.Sprintf("%s://%s%s", scheme, u.Host, repoPath) + } + + return repoURL, ref, chartPath, nil +} + +// computeHash computes a SHA256 hash for the given data +func computeHash(data []byte) string { + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]) +} + +// NewGitGetter constructs a valid Git backend handler +func NewGitGetter(options ...Option) (Getter, error) { + var client GitGetter + + for _, opt := range options { + opt(&client.opts) + } + + // Check if git is available + if _, err := exec.LookPath("git"); err != nil { + return nil, fmt.Errorf("git command not found in PATH: %w", err) + } + + // Check if helm is available (for packaging) + if _, err := exec.LookPath("helm"); err != nil { + return nil, fmt.Errorf("helm command not found in PATH: %w", err) + } + + return &client, nil +} diff --git a/pkg/getter/gitgetter_test.go b/pkg/getter/gitgetter_test.go new file mode 100644 index 000000000..29fb70411 --- /dev/null +++ b/pkg/getter/gitgetter_test.go @@ -0,0 +1,142 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package getter + +import ( + "testing" +) + +func TestNewGitGetter(t *testing.T) { + g, err := NewGitGetter() + if err != nil { + t.Skip("git or helm command not found in PATH, skipping test") + } + + if _, ok := g.(*GitGetter); !ok { + t.Fatal("Expected NewGitGetter to produce a *GitGetter") + } +} + +func TestParseGitURL(t *testing.T) { + tests := []struct { + name string + url string + expectRepo string + expectRef string + expectPath string + expectError bool + }{ + { + name: "git:// with ref and path", + url: "git://github.com/user/repo@v1.0.0?path=charts/mychart", + expectRepo: "https://github.com/user/repo", + expectRef: "v1.0.0", + expectPath: "charts/mychart", + expectError: false, + }, + { + name: "git+https:// with ref", + url: "git+https://github.com/user/repo@main", + expectRepo: "https://github.com/user/repo", + expectRef: "main", + expectPath: "", + expectError: false, + }, + { + name: "git+https:// without ref", + url: "git+https://github.com/user/repo", + expectRepo: "https://github.com/user/repo", + expectRef: "HEAD", + expectPath: "", + expectError: false, + }, + { + name: "git+ssh:// with ref and path", + url: "git+ssh://git@github.com/user/repo@v2.0.0?path=charts/test", + expectRepo: "git@github.com:user/repo", + expectRef: "v2.0.0", + expectPath: "charts/test", + expectError: false, + }, + { + name: "git:// with ref query param", + url: "git://github.com/user/repo?ref=feature-branch&path=charts/app", + expectRepo: "https://github.com/user/repo", + expectRef: "feature-branch", + expectPath: "charts/app", + expectError: false, + }, + { + name: "git+http:// simple", + url: "git+http://gitlab.com/group/project@develop", + expectRepo: "http://gitlab.com/group/project", + expectRef: "develop", + expectPath: "", + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo, ref, path, err := parseGitURL(tt.url) + + if tt.expectError { + if err == nil { + t.Errorf("expected error but got none") + } + return + } + + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + if repo != tt.expectRepo { + t.Errorf("expected repo %q, got %q", tt.expectRepo, repo) + } + + if ref != tt.expectRef { + t.Errorf("expected ref %q, got %q", tt.expectRef, ref) + } + + if path != tt.expectPath { + t.Errorf("expected path %q, got %q", tt.expectPath, path) + } + }) + } +} + +func TestGitGetterSchemes(t *testing.T) { + // Test that Git getter is registered for the correct schemes + schemes := []string{"git", "git+https", "git+http", "git+ssh"} + + providers := Getters() + + for _, scheme := range schemes { + found := false + for _, p := range providers { + if p.Provides(scheme) { + found = true + break + } + } + if !found { + t.Errorf("scheme %q not registered with any provider", scheme) + } + } +}