From acdbb0504ad5370ef9c052a96290e153b0b6968a Mon Sep 17 00:00:00 2001 From: yxxhero Date: Tue, 16 Mar 2021 19:38:29 +0800 Subject: [PATCH] Implement the Getter interface for the git:// protocol. Signed-off-by: yxxhero --- cmd/helm/dependency.go | 10 +-- internal/fileutil/fileutil.go | 59 ++++++++++++++++ pkg/downloader/chart_downloader.go | 24 ++++++- pkg/downloader/manager.go | 35 ++++++---- pkg/getter/getter.go | 13 +++- pkg/getter/getter_test.go | 4 +- .../git_downloader.go => getter/gitgetter.go} | 68 +++++++++++++------ pkg/getter/gitgetter_test.go | 31 +++++++++ 8 files changed, 200 insertions(+), 44 deletions(-) rename pkg/{downloader/git_downloader.go => getter/gitgetter.go} (52%) create mode 100644 pkg/getter/gitgetter_test.go diff --git a/cmd/helm/dependency.go b/cmd/helm/dependency.go index 75205acae..d6799a4e4 100644 --- a/cmd/helm/dependency.go +++ b/cmd/helm/dependency.go @@ -75,11 +75,11 @@ for this case. A repository can be defined as a git URL. The path must start with a prefix of "git:" followed by a valid git repository URL. -# requirements.yaml -dependencies: -- name: nginx - version: "master" - repository: "git:https://github.com/helm/helm-chart.git" + # Chart.yaml + dependencies: + - name: nginx + version: "master" + repository: "git:https://github.com/helm/helm-chart.git" The 'repository' can be the https or ssh URL that you would use to clone a git repo or add as a git remote, prefixed with 'git:'. diff --git a/internal/fileutil/fileutil.go b/internal/fileutil/fileutil.go index 739093f3b..0ca7bdf67 100644 --- a/internal/fileutil/fileutil.go +++ b/internal/fileutil/fileutil.go @@ -21,6 +21,11 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" + + "archive/tar" + "bytes" + "compress/gzip" "helm.sh/helm/v3/internal/third_party/dep/fs" ) @@ -49,3 +54,57 @@ func AtomicWriteFile(filename string, reader io.Reader, mode os.FileMode) error return fs.RenameWithFallback(tempName, filename) } + +func CompressDirToTgz(src, tmpdir string) (*bytes.Buffer, error) { + // tar => gzip => buf + + buf := bytes.NewBuffer(nil) + zr := gzip.NewWriter(buf) + tw := tar.NewWriter(zr) + + // walk through every file in the folder + walkErr := filepath.Walk(src, func(file string, fi os.FileInfo, err error) error { + + // generate tar header + if err != nil { + return err + } + header, err := tar.FileInfoHeader(fi, strings.TrimPrefix(file, tmpdir+"/")) + if err != nil { + return err + } + + // must provide real name + // (see https://golang.org/src/archive/tar/common.go?#L626) + header.Name = strings.TrimPrefix(filepath.ToSlash(file), tmpdir+"/") + + // write header + if err := tw.WriteHeader(header); err != nil { + return err + } + // if not a dir, write file content + if !fi.IsDir() { + data, err := os.Open(file) + if err != nil { + return err + } + if _, err := io.Copy(tw, data); err != nil { + return err + } + } + return nil + }) + if walkErr != nil { + return nil, walkErr + } + + // produce tar + if err := tw.Close(); err != nil { + return nil, err + } + // produce gzip + if err := zr.Close(); err != nil { + return nil, err + } + return buf, nil +} diff --git a/pkg/downloader/chart_downloader.go b/pkg/downloader/chart_downloader.go index 3feb5b702..23bd50ed6 100644 --- a/pkg/downloader/chart_downloader.go +++ b/pkg/downloader/chart_downloader.go @@ -97,13 +97,20 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven return "", nil, err } - data, err := g.Get(u.String(), c.Options...) + downloadURL := "" + if u.Scheme == "git" { + downloadURL = u.Host + u.Path + } else { + downloadURL = u.String() + } + + data, err := g.Get(downloadURL, c.Options...) if err != nil { return "", nil, err } name := filepath.Base(u.Path) - if u.Scheme == registry.OCIScheme { + if u.Scheme == registry.OCIScheme || u.Scheme == "git" { idx := strings.LastIndexByte(name, ':') name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) } @@ -190,6 +197,19 @@ func (c *ChartDownloader) getOciURI(ref, version string, u *url.URL) (*url.URL, // * If version is empty, this will return the URL for the latest version // * If no version can be found, an error is returned func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, error) { + if strings.HasPrefix(ref, "git:") { + gitRefSplitResult := strings.Split(ref, "git:") + gitURL := gitRefSplitResult[1] + u, err := url.Parse(gitURL) + if err != nil { + return nil, errors.Errorf("invalid git URL format: %s", gitURL) + } + return &url.URL{ + Scheme: "git", + Host: u.Scheme + "://" + u.Host, + Path: u.Path, + }, nil + } u, err := url.Parse(ref) if err != nil { return nil, errors.Errorf("invalid chart URL format: %s", ref) diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index ffbcb4c11..4222d3d96 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -310,20 +310,6 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { dep.Version = ver continue } - if strings.HasPrefix(dep.Repository, "git:") { - destPath := filepath.Join(m.ChartPath, "charts") - gitURL := strings.TrimPrefix(dep.Repository, "git:") - if m.Debug { - fmt.Fprintf(m.Out, "Downloading %s from git repo %s\n", dep.Name, gitURL) - } - dl := GitDownloader{} - if err := dl.DownloadTo(gitURL, dep.Version, destPath); err != nil { - saveError = fmt.Errorf("could not download %s: %s", gitURL, err) - break - } - continue - } - fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) // Any failure to resolve/download a chart should fail: // https://github.com/helm/helm/issues/1439 @@ -357,7 +343,13 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { } version := "" + if registry.IsOCI(churl) { + if !resolver.FeatureGateOCI.IsEnabled() { + return errors.Wrapf(resolver.FeatureGateOCI.Error(), + "the repository %s is an OCI registry", churl) + } + churl, version, err = parseOCIRef(churl) if err != nil { return errors.Wrapf(err, "could not parse OCI reference") @@ -367,6 +359,17 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { getter.WithTagName(version)) } + if strings.HasPrefix(churl, "git://") { + version = dep.Version + + dl.Options = append(dl.Options, getter.WithTagName(version)) + dl.Options = append(dl.Options, getter.WithChartName(dep.Name)) + + if m.Debug { + fmt.Fprintf(m.Out, "Downloading %s from git repo %s\n", dep.Name, churl) + } + } + if _, _, err = dl.DownloadTo(churl, version, tmpPath); err != nil { saveError = errors.Wrapf(err, "could not download %s", churl) break @@ -733,6 +736,10 @@ func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error { // // If it finds a URL that is "relative", it will prepend the repoURL. func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (url, username, password string, insecureskiptlsverify, passcredentialsall bool, caFile, certFile, keyFile string, err error) { + if strings.HasPrefix(repoURL, "git://") { + return repoURL, "", "", false, false, "", "", "", nil + } + if registry.IsOCI(repoURL) { return fmt.Sprintf("%s/%s:%s", repoURL, name, version), "", "", false, false, "", "", "", nil } diff --git a/pkg/getter/getter.go b/pkg/getter/getter.go index 653b032fe..489a7f6c9 100644 --- a/pkg/getter/getter.go +++ b/pkg/getter/getter.go @@ -42,6 +42,7 @@ type options struct { passCredentialsAll bool userAgent string version string + chartName string registryClient *registry.Client timeout time.Duration transport *http.Transport @@ -66,6 +67,11 @@ func WithBasicAuth(username, password string) Option { opts.password = password } } +func WithChartName(chartName string) Option { + return func(opts *options) { + opts.chartName = chartName + } +} func WithPassCredentialsAll(pass bool) Option { return func(opts *options) { @@ -182,11 +188,16 @@ var ociProvider = Provider{ New: NewOCIGetter, } +var gitProvider = Provider{ + Schemes: []string{"git"}, + New: NewGITGetter, +} + // All finds all of the registered getters as a list of Provider instances. // Currently, the built-in getters and the discovered plugins with downloader // notations are collected. func All(settings *cli.EnvSettings) Providers { - result := Providers{httpProvider, ociProvider} + result := Providers{httpProvider, ociProvider, gitProvider} pluginDownloaders, _ := collectPlugins(settings) result = append(result, pluginDownloaders...) return result diff --git a/pkg/getter/getter_test.go b/pkg/getter/getter_test.go index ab14784ab..c331f2c0f 100644 --- a/pkg/getter/getter_test.go +++ b/pkg/getter/getter_test.go @@ -57,8 +57,8 @@ func TestAll(t *testing.T) { env.PluginsDirectory = pluginDir all := All(env) - if len(all) != 4 { - t.Errorf("expected 4 providers (default plus three plugins), got %d", len(all)) + if len(all) != 5 { + t.Errorf("expected 5 providers (default plus three plugins), got %d", len(all)) } if _, err := all.ByScheme("test2"); err != nil { diff --git a/pkg/downloader/git_downloader.go b/pkg/getter/gitgetter.go similarity index 52% rename from pkg/downloader/git_downloader.go rename to pkg/getter/gitgetter.go index 28f7c42fe..c4b46a530 100644 --- a/pkg/downloader/git_downloader.go +++ b/pkg/getter/gitgetter.go @@ -13,28 +13,32 @@ See the License for the specific language governing permissions and limitations under the License. */ -package downloader +package getter import ( + "bytes" + "strings" + "fmt" "io/ioutil" "os" "path/filepath" - "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/internal/fileutil" "helm.sh/helm/v3/pkg/gitutil" ) // Assigned here so it can be overridden for testing. var gitCloneTo = gitutil.CloneTo -// GitDownloader handles downloading a chart from a git url. -type GitDownloader struct{} +// GITGetter is the default HTTP(/S) backend handler +type GITGetter struct { + opts options +} // ensureGitDirIgnored will append ".git/" to the .helmignore file in a directory. // Create the .helmignore file if it does not exist. -func (g *GitDownloader) ensureGitDirIgnored(repoPath string) error { +func (g *GITGetter) ensureGitDirIgnored(repoPath string) error { helmignorePath := filepath.Join(repoPath, ".helmignore") f, err := os.OpenFile(helmignorePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { @@ -47,17 +51,34 @@ func (g *GitDownloader) ensureGitDirIgnored(repoPath string) error { return nil } -// DownloadTo will create a temp directory, then fetch a git repo into it. -// The git repo will be archived into a chart and copied to the destPath. -func (g *GitDownloader) DownloadTo(gitURL string, ref string, destPath string) error { +//Get performs a Get from repo.Getter and returns the body. +func (g *GITGetter) Get(href string, options ...Option) (*bytes.Buffer, error) { + for _, opt := range options { + opt(&g.opts) + } + return g.get(href) +} + +func (g *GITGetter) get(href string) (*bytes.Buffer, error) { + gitURL := strings.TrimPrefix(href, "git:") + version := g.opts.version + chartName := g.opts.chartName + if version == "" { + return nil, fmt.Errorf("The version must be a valid tag or branch name for the git repo, not nil") + } tmpDir, err := ioutil.TempDir("", "helm") if err != nil { - return err + return nil, err + } + chartTmpDir := filepath.Join(tmpDir, chartName) + + if err := os.MkdirAll(chartTmpDir, 0755); err != nil { + return nil, err } defer os.RemoveAll(tmpDir) - if err = gitCloneTo(gitURL, ref, tmpDir); err != nil { - return fmt.Errorf("Unable to retrieve git repo. %s", err) + if err = gitCloneTo(gitURL, version, chartTmpDir); err != nil { + return nil, fmt.Errorf("Unable to retrieve git repo. %s", err) } // A .helmignore that includes an ignore for .git/ should be included in the git repo itself, @@ -65,14 +86,21 @@ func (g *GitDownloader) DownloadTo(gitURL string, ref string, destPath string) e // To prevent the git history from bleeding into the charts archive, append/create .helmignore. g.ensureGitDirIgnored(tmpDir) - // Turn the extracted git archive into a chart and move it into the charts directory. - // This is using chartutil.Save() so that .helmignore logic is applied. - loadedChart, loadErr := loader.LoadDir(tmpDir) - if loadErr != nil { - return fmt.Errorf("Unable to process the git repo %s as a chart. %s", gitURL, err) + buf, err := fileutil.CompressDirToTgz(chartTmpDir, tmpDir) + if err != nil { + return nil, fmt.Errorf("Unable to tar and compress dir %s to tgz file. %s", tmpDir, err) } - if _, saveErr := chartutil.Save(loadedChart, destPath); saveErr != nil { - return fmt.Errorf("Unable to save the git repo %s as a chart. %s", gitURL, err) + return buf, nil +} + +// NewGITGetter constructs a valid http/https client as a Getter +func NewGITGetter(ops ...Option) (Getter, error) { + + client := GITGetter{} + + for _, opt := range ops { + opt(&client.opts) } - return nil + + return &client, nil } diff --git a/pkg/getter/gitgetter_test.go b/pkg/getter/gitgetter_test.go new file mode 100644 index 000000000..cf82edbb3 --- /dev/null +++ b/pkg/getter/gitgetter_test.go @@ -0,0 +1,31 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package getter + +import ( + "testing" +) + +func TestNewGITGetter(t *testing.T) { + g, err := NewGITGetter() + if err != nil { + t.Fatal(err) + } + + if _, ok := g.(*GITGetter); !ok { + t.Fatal("Expected NewGITGetter to produce an *GITGetter") + } +}