diff --git a/cmd/helm/dependency.go b/cmd/helm/dependency.go index 2cc4c5045..0b96348d3 100644 --- a/cmd/helm/dependency.go +++ b/cmd/helm/dependency.go @@ -71,6 +71,15 @@ the dependency charts stored locally. The path should start with a prefix of If the dependency chart is retrieved locally, it is not required to have the repository added to helm by "helm add repo". Version matching is also supported for this case. + +Starting from 3.3.0, if OCI Registry support has been enabled via the HELM_EXPERIMENTAL_OCI +flag, repository can be defined as an OCI image reference. The path should start with a +prefix of "oci://". For example, + + # Chart.yaml + dependencies: + - version: "1.2.3" + repository: "oci://localhost:5000/myrepo/mychart:2.7.0" ` const dependencyListDesc = ` @@ -82,7 +91,7 @@ the contents of a chart. This will produce an error if the chart cannot be loaded. ` -func newDependencyCmd(out io.Writer) *cobra.Command { +func newDependencyCmd(cfg *action.Configuration, out io.Writer) *cobra.Command { cmd := &cobra.Command{ Use: "dependency update|build|list", Aliases: []string{"dep", "dependencies"}, @@ -92,8 +101,8 @@ func newDependencyCmd(out io.Writer) *cobra.Command { } cmd.AddCommand(newDependencyListCmd(out)) - cmd.AddCommand(newDependencyUpdateCmd(out)) - cmd.AddCommand(newDependencyBuildCmd(out)) + cmd.AddCommand(newDependencyUpdateCmd(cfg, out)) + cmd.AddCommand(newDependencyBuildCmd(cfg, out)) return cmd } diff --git a/cmd/helm/dependency_build.go b/cmd/helm/dependency_build.go index 478b49479..3b18daabb 100644 --- a/cmd/helm/dependency_build.go +++ b/cmd/helm/dependency_build.go @@ -24,6 +24,7 @@ import ( "k8s.io/client-go/util/homedir" "helm.sh/helm/v3/cmd/helm/require" + "helm.sh/helm/v3/internal/experimental/registry" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/downloader" "helm.sh/helm/v3/pkg/getter" @@ -40,7 +41,7 @@ If no lock file is found, 'helm dependency build' will mirror the behavior of 'helm dependency update'. ` -func newDependencyBuildCmd(out io.Writer) *cobra.Command { +func newDependencyBuildCmd(cfg *action.Configuration, out io.Writer) *cobra.Command { client := action.NewDependency() cmd := &cobra.Command{ @@ -53,11 +54,15 @@ func newDependencyBuildCmd(out io.Writer) *cobra.Command { if len(args) > 0 { chartpath = filepath.Clean(args[0]) } + getters := getter.All(settings) + if FeatureGateOCI.IsEnabled() { + getters = append(getters, registry.NewRegistryGetterProvider(cfg.RegistryClient)) + } man := &downloader.Manager{ Out: out, ChartPath: chartpath, Keyring: client.Keyring, - Getters: getter.All(settings), + Getters: getters, RepositoryConfig: settings.RepositoryConfig, RepositoryCache: settings.RepositoryCache, Debug: settings.Debug, diff --git a/cmd/helm/dependency_update.go b/cmd/helm/dependency_update.go index 9855afb92..142ebd6d2 100644 --- a/cmd/helm/dependency_update.go +++ b/cmd/helm/dependency_update.go @@ -22,6 +22,7 @@ import ( "github.com/spf13/cobra" "helm.sh/helm/v3/cmd/helm/require" + "helm.sh/helm/v3/internal/experimental/registry" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/downloader" "helm.sh/helm/v3/pkg/getter" @@ -43,7 +44,7 @@ in the Chart.yaml file, but (b) at the wrong version. ` // newDependencyUpdateCmd creates a new dependency update command. -func newDependencyUpdateCmd(out io.Writer) *cobra.Command { +func newDependencyUpdateCmd(cfg *action.Configuration, out io.Writer) *cobra.Command { client := action.NewDependency() cmd := &cobra.Command{ @@ -57,12 +58,16 @@ func newDependencyUpdateCmd(out io.Writer) *cobra.Command { if len(args) > 0 { chartpath = filepath.Clean(args[0]) } + getters := getter.All(settings) + if FeatureGateOCI.IsEnabled() { + getters = append(getters, registry.NewRegistryGetterProvider(cfg.RegistryClient)) + } man := &downloader.Manager{ Out: out, ChartPath: chartpath, Keyring: client.Keyring, SkipUpdate: client.SkipRefresh, - Getters: getter.All(settings), + Getters: getters, RepositoryConfig: settings.RepositoryConfig, RepositoryCache: settings.RepositoryCache, Debug: settings.Debug, diff --git a/cmd/helm/root.go b/cmd/helm/root.go index 69f5855d9..686e45036 100644 --- a/cmd/helm/root.go +++ b/cmd/helm/root.go @@ -141,7 +141,7 @@ func newRootCmd(actionConfig *action.Configuration, out io.Writer, args []string cmd.AddCommand( // chart commands newCreateCmd(out), - newDependencyCmd(out), + newDependencyCmd(actionConfig, out), newPullCmd(out), newShowCmd(out), newLintCmd(out), diff --git a/go.mod b/go.mod index f23d152a2..6467b03d5 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/mitchellh/copystructure v1.0.0 github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 + github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pkg/errors v0.9.1 github.com/rubenv/sql-migrate v0.0.0-20200212082348-64f95ea68aa3 github.com/sirupsen/logrus v1.4.2 diff --git a/internal/experimental/registry/client_test.go b/internal/experimental/registry/client_test.go index 2d208b7b9..459a8a9b3 100644 --- a/internal/experimental/registry/client_test.go +++ b/internal/experimental/registry/client_test.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "io/ioutil" - "net" "net/http" "net/http/httptest" "net/url" @@ -33,12 +32,12 @@ import ( "time" "github.com/containerd/containerd/errdefs" - auth "github.com/deislabs/oras/pkg/auth/docker" "github.com/docker/distribution/configuration" "github.com/docker/distribution/registry" _ "github.com/docker/distribution/registry/auth/htpasswd" _ "github.com/docker/distribution/registry/storage/driver/inmemory" + "github.com/phayes/freeport" "github.com/stretchr/testify/suite" "golang.org/x/crypto/bcrypt" @@ -107,7 +106,7 @@ func (suite *RegistryClientTestSuite) SetupSuite() { // Registry config config := &configuration.Configuration{} - port, err := getFreePort() + port, err := freeport.GetFreePort() suite.Nil(err, "no error finding free port for test registry") suite.DockerRegistryHost = fmt.Sprintf("localhost:%d", port) config.HTTP.Addr = fmt.Sprintf(":%d", port) @@ -254,21 +253,6 @@ func TestRegistryClientTestSuite(t *testing.T) { suite.Run(t, new(RegistryClientTestSuite)) } -// borrowed from https://github.com/phayes/freeport -func getFreePort() (int, error) { - addr, err := net.ResolveTCPAddr("tcp", "localhost:0") - if err != nil { - return 0, err - } - - l, err := net.ListenTCP("tcp", addr) - if err != nil { - return 0, err - } - defer l.Close() - return l.Addr().(*net.TCPAddr).Port, nil -} - func initCompromisedRegistryTestServer() string { s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.Contains(r.URL.Path, "manifests") { diff --git a/internal/experimental/registry/constants.go b/internal/experimental/registry/constants.go index dafb3c9e5..556e8771e 100644 --- a/internal/experimental/registry/constants.go +++ b/internal/experimental/registry/constants.go @@ -22,6 +22,9 @@ const ( // HelmChartContentLayerMediaType is the reserved media type for Helm chart package content HelmChartContentLayerMediaType = "application/tar+gzip" + + // OCIProtocol is the protocol used for OCI registry URLs + OCIProtocol = "oci" ) // KnownMediaTypes returns a list of layer mediaTypes that the Helm client knows about diff --git a/internal/experimental/registry/getter.go b/internal/experimental/registry/getter.go new file mode 100644 index 000000000..af2cae4f0 --- /dev/null +++ b/internal/experimental/registry/getter.go @@ -0,0 +1,78 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry // import "helm.sh/helm/v3/internal/experimental/registry" + +import ( + "bytes" + "net/url" + + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/getter" +) + +// Getter is the HTTP(/S) backend handler for OCI image registries. +type Getter struct { + Client *Client +} + +func NewRegistryGetter(c *Client) *Getter { + return &Getter{Client: c} +} + +func NewRegistryGetterProvider(c *Client) getter.Provider { + return getter.Provider{ + Schemes: []string{OCIProtocol}, + New: func(options ...getter.Option) (g getter.Getter, e error) { + return NewRegistryGetter(c), nil + }, + } +} + +func (g *Getter) Get(href string, options ...getter.Option) (*bytes.Buffer, error) { + u, err := url.Parse(href) + + if err != nil { + return nil, err + } + + ref, err := ParseReference(u.Host + u.Path) + + if err != nil { + return nil, err + } + + // first we'll pull the chart + err = g.Client.PullChart(ref) + + if err != nil { + return nil, err + } + + // once we know we have the chart, we'll load up the chart + c, err := g.Client.LoadChart(ref) + + if err != nil { + return nil, err + } + + buf := bytes.NewBuffer(nil) + + // lastly, we'll write the tarred and gzipped contents of the chart to our output buffer + err = chartutil.Write(c, buf) + + return buf, err +} diff --git a/internal/experimental/registry/getter_test.go b/internal/experimental/registry/getter_test.go new file mode 100644 index 000000000..af27a7297 --- /dev/null +++ b/internal/experimental/registry/getter_test.go @@ -0,0 +1,129 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + auth "github.com/deislabs/oras/pkg/auth/docker" + "github.com/docker/distribution/configuration" + "github.com/docker/distribution/registry" + "github.com/phayes/freeport" + "github.com/stretchr/testify/assert" + "golang.org/x/crypto/bcrypt" + + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" +) + +func TestValidRegistryUrlWithImageTag(t *testing.T) { + os.RemoveAll(testCacheRootDir) + os.Mkdir(testCacheRootDir, 0700) + + var out bytes.Buffer + credentialsFile := filepath.Join(testCacheRootDir, CredentialsFileBasename) + + client, err := auth.NewClient(credentialsFile) + assert.Nil(t, err, "no error creating auth client") + + resolver, err := client.Resolver(context.Background(), http.DefaultClient, false) + assert.Nil(t, err, "no error creating resolver") + + // create cache + cache, err := NewCache( + CacheOptDebug(true), + CacheOptWriter(&out), + CacheOptRoot(filepath.Join(testCacheRootDir, CacheRootDir)), + ) + assert.Nil(t, err, "no error creating cache") + + // init test client + registryClient, err := NewClient( + ClientOptDebug(true), + ClientOptWriter(&out), + ClientOptAuthorizer(&Authorizer{ + Client: client, + }), + ClientOptResolver(&Resolver{ + Resolver: resolver, + }), + ClientOptCache(cache), + ) + assert.Nil(t, err, "no error creating registry client") + + // create htpasswd file (w BCrypt, which is required) + pwBytes, err := bcrypt.GenerateFromPassword([]byte(testPassword), bcrypt.DefaultCost) + assert.Nil(t, err, "no error generating bcrypt password for test htpasswd file") + htpasswdPath := filepath.Join(testCacheRootDir, testHtpasswdFileBasename) + err = ioutil.WriteFile(htpasswdPath, []byte(fmt.Sprintf("%s:%s\n", testUsername, string(pwBytes))), 0644) + assert.Nil(t, err, "no error creating test htpasswd file") + + // Registry config + config := &configuration.Configuration{} + port, err := freeport.GetFreePort() + assert.Nil(t, err, "failed to find free port for test registry") + dockerRegistryHost := fmt.Sprintf("localhost:%d", port) + config.HTTP.Addr = fmt.Sprintf(":%d", port) + config.HTTP.DrainTimeout = time.Duration(10) * time.Second + config.Storage = map[string]configuration.Parameters{"inmemory": map[string]interface{}{}} + config.Auth = configuration.Auth{ + "htpasswd": configuration.Parameters{ + "realm": "localhost", + "path": htpasswdPath, + }, + } + dockerRegistry, err := registry.NewRegistry(context.Background(), config) + assert.Nil(t, err, "failed to create test registry") + + // Start Docker registry + go dockerRegistry.ListenAndServe() + registryClient.Login(dockerRegistryHost, testUsername, testPassword, false) + + ref, _ := ParseReference(fmt.Sprintf("%s/testrepo/testchart:1.2.3", dockerRegistryHost)) + + ch := &chart.Chart{} + ch.Metadata = &chart.Metadata{ + APIVersion: "v1", + Name: "testchart", + Version: "1.2.3", + } + + err = registryClient.SaveChart(ch, ref) + assert.NoError(t, err) + err = registryClient.PushChart(ref) + assert.NoError(t, err) + + g := NewRegistryGetter(registryClient) + res, err := g.Get(fmt.Sprintf("oci://%s/testrepo/testchart:1.2.3", dockerRegistryHost)) + assert.NoError(t, err, "failed to retrieve chart") + + downloadedChart, err := loader.LoadArchive(res) + assert.NoError(t, err, "failed to load archive") + assert.Equal(t, "testchart", downloadedChart.Name()) + assert.Equal(t, "1.2.3", downloadedChart.Metadata.Version) + + registryClient.Logout(dockerRegistryHost) + os.RemoveAll(testCacheRootDir) +} diff --git a/pkg/chartutil/save.go b/pkg/chartutil/save.go index 2ce4eddaf..6653395be 100644 --- a/pkg/chartutil/save.go +++ b/pkg/chartutil/save.go @@ -21,6 +21,7 @@ import ( "compress/gzip" "encoding/json" "fmt" + "io" "os" "path/filepath" "time" @@ -124,28 +125,50 @@ func Save(c *chart.Chart, outDir string) (string, error) { return "", err } + rollback := false + // save the chart to the file + err = Write(c, f) + + if err != nil { + rollback = true + return filename, err + } + + defer func() { + f.Close() + if rollback { + os.Remove(filename) + } + }() + + return filename, nil +} + +// Write streams an archived chart to an io.writer interface. +// +// This takes an existing chart and a destination writer. +func Write(c *chart.Chart, w io.Writer) error { + if err := c.Validate(); err != nil { + return errors.Wrap(err, "chart validation") + } + // Wrap in gzip writer - zipper := gzip.NewWriter(f) + zipper := gzip.NewWriter(w) zipper.Header.Extra = headerBytes zipper.Header.Comment = "Helm" // Wrap in tar writer twriter := tar.NewWriter(zipper) - rollback := false defer func() { twriter.Close() zipper.Close() - f.Close() - if rollback { - os.Remove(filename) - } }() if err := writeTarContents(twriter, c, ""); err != nil { - rollback = true - return filename, err + return err } - return filename, nil + + return nil } func writeTarContents(out *tar.Writer, c *chart.Chart, prefix string) error { diff --git a/pkg/downloader/chart_downloader.go b/pkg/downloader/chart_downloader.go index ef26f3348..fa8e56d57 100644 --- a/pkg/downloader/chart_downloader.go +++ b/pkg/downloader/chart_downloader.go @@ -25,6 +25,7 @@ import ( "github.com/pkg/errors" + "helm.sh/helm/v3/internal/experimental/registry" "helm.sh/helm/v3/internal/fileutil" "helm.sh/helm/v3/internal/urlutil" "helm.sh/helm/v3/pkg/getter" @@ -94,12 +95,31 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven return "", nil, err } - data, err := g.Get(u.String(), c.Options...) + downloadURL := u.String() + name := filepath.Base(u.Path) + + if _, ok := g.(*registry.Getter); ok { + parts := strings.Split(filepath.Base(u.Path), ":") + + if len(parts) == 1 && version == "" { + return "", nil, errors.New("no version or tag provided") + } + + if len(parts) != 2 { + parts = append(parts, version) + u.Path = fmt.Sprintf("%s:%s", u.Path, version) + } + + downloadURL = u.String() + name = fmt.Sprintf("%s-%s.tgz", parts[0], parts[1]) + } + + data, err := g.Get(downloadURL, c.Options...) + if err != nil { return "", nil, err } - name := filepath.Base(u.Path) destfile := filepath.Join(dest, name) if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil { return destfile, nil, err diff --git a/pkg/downloader/chart_downloader_test.go b/pkg/downloader/chart_downloader_test.go index abfb007ff..bf232ccce 100644 --- a/pkg/downloader/chart_downloader_test.go +++ b/pkg/downloader/chart_downloader_test.go @@ -16,12 +16,28 @@ limitations under the License. package downloader import ( + "bytes" + "context" + "fmt" + "io/ioutil" "net/http" "os" "path/filepath" "testing" - + "time" + + auth "github.com/deislabs/oras/pkg/auth/docker" + "github.com/docker/distribution/configuration" + "github.com/docker/distribution/registry" + _ "github.com/docker/distribution/registry/auth/htpasswd" + _ "github.com/docker/distribution/registry/storage/driver/inmemory" + "github.com/phayes/freeport" + "github.com/stretchr/testify/assert" + "golang.org/x/crypto/bcrypt" + + helmregistry "helm.sh/helm/v3/internal/experimental/registry" "helm.sh/helm/v3/internal/test/ensure" + "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/repo" @@ -322,6 +338,298 @@ func TestDownloadTo_VerifyLater(t *testing.T) { } } +func TestDownloadToFromOCIRepository(t *testing.T) { + var ( + CredentialsFileBasename = "config.json" + testCacheRootDir = "helm-registry-test" + testHtpasswdFileBasename = "authtest.htpasswd" + testUsername = "myuser" + testPassword = "mypass" + ) + os.RemoveAll(testCacheRootDir) + os.Mkdir(testCacheRootDir, 0700) + + var out bytes.Buffer + credentialsFile := filepath.Join(testCacheRootDir, CredentialsFileBasename) + + client, err := auth.NewClient(credentialsFile) + assert.Nil(t, err, "no error creating auth client") + + resolver, err := client.Resolver(context.Background(), http.DefaultClient, false) + assert.Nil(t, err, "no error creating resolver") + + // create cache + cache, err := helmregistry.NewCache( + helmregistry.CacheOptDebug(true), + helmregistry.CacheOptWriter(&out), + helmregistry.CacheOptRoot(filepath.Join(testCacheRootDir, helmregistry.CacheRootDir)), + ) + + assert.Nil(t, err, "failed creating cache") + + // init test client + registryClient, err := helmregistry.NewClient( + helmregistry.ClientOptDebug(true), + helmregistry.ClientOptWriter(&out), + helmregistry.ClientOptAuthorizer(&helmregistry.Authorizer{ + Client: client, + }), + helmregistry.ClientOptResolver(&helmregistry.Resolver{ + Resolver: resolver, + }), + helmregistry.ClientOptCache(cache), + ) + assert.Nil(t, err, "failed creating registry client") + + // create htpasswd file (w BCrypt, which is required) + pwBytes, err := bcrypt.GenerateFromPassword([]byte(testPassword), bcrypt.DefaultCost) + assert.Nil(t, err, "no error generating bcrypt password for test htpasswd file") + htpasswdPath := filepath.Join(testCacheRootDir, testHtpasswdFileBasename) + err = ioutil.WriteFile(htpasswdPath, []byte(fmt.Sprintf("%s:%s\n", testUsername, string(pwBytes))), 0644) + assert.Nil(t, err, "error creating test htpasswd file") + + // Registry config + config := &configuration.Configuration{} + port, err := freeport.GetFreePort() + assert.Nil(t, err, "no error finding free port for test registry") + dockerRegistryHost := fmt.Sprintf("localhost:%d", port) + config.HTTP.Addr = fmt.Sprintf(":%d", port) + config.HTTP.DrainTimeout = time.Duration(10) * time.Second + config.Storage = map[string]configuration.Parameters{"inmemory": map[string]interface{}{}} + config.Auth = configuration.Auth{ + "htpasswd": configuration.Parameters{ + "realm": "localhost", + "path": htpasswdPath, + }, + } + dockerRegistry, err := registry.NewRegistry(context.Background(), config) + assert.Nil(t, err, "no error creating test registry") + + // Start Docker registry + go dockerRegistry.ListenAndServe() + err = registryClient.Login(dockerRegistryHost, testUsername, testPassword, false) + assert.Nil(t, err, "failed to login to registry with username "+testUsername+" and password "+testPassword) + + ref, _ := helmregistry.ParseReference(fmt.Sprintf("%s/testrepo/testchart:1.2.3", dockerRegistryHost)) + ch, err := loader.LoadDir("testdata/local-subchart") + assert.Nil(t, err, "failed to load local chart") + err = registryClient.SaveChart(ch, ref) + assert.Nil(t, err, "failed to save chart") + err = registryClient.PushChart(ref) + assert.Nil(t, err, "failed to push chart") + + c := ChartDownloader{ + Out: os.Stderr, + Verify: VerifyIfPossible, + Keyring: "testdata/helm-test-key.pub", + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + Getters: append(getter.All(&cli.EnvSettings{ + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + }), helmregistry.NewRegistryGetterProvider(registryClient)), + Options: []getter.Option{ + getter.WithBasicAuth("username", "password"), + }, + } + // the filename becomes the {last segment of the image name}-{the image tag} + fname := "/testchart-1.2.3.tgz" + dest := ensure.TempDir(t) + where, _, err := c.DownloadTo(fmt.Sprintf("oci://%s/testrepo/testchart:1.2.3", dockerRegistryHost), "", dest) + if err != nil { + t.Fatal(err) + } + + if expect := filepath.Join(dest, fname); where != expect { + t.Errorf("Expected download to %s, got %s", expect, where) + } + + if _, err := os.Stat(filepath.Join(dest, fname)); err != nil { + t.Error(err) + } + + os.RemoveAll(testCacheRootDir) +} + +func TestDownloadToFromOCIRepositoryWithoutTag(t *testing.T) { + var ( + CredentialsFileBasename = "config.json" + testCacheRootDir = "helm-registry-test" + testHtpasswdFileBasename = "authtest.htpasswd" + testUsername = "myuser" + testPassword = "mypass" + ) + os.RemoveAll(testCacheRootDir) + os.Mkdir(testCacheRootDir, 0700) + + var out bytes.Buffer + credentialsFile := filepath.Join(testCacheRootDir, CredentialsFileBasename) + + client, err := auth.NewClient(credentialsFile) + assert.Nil(t, err, "no error creating auth client") + + resolver, err := client.Resolver(context.Background(), http.DefaultClient, false) + assert.Nil(t, err, "no error creating resolver") + + // create cache + cache, err := helmregistry.NewCache( + helmregistry.CacheOptDebug(true), + helmregistry.CacheOptWriter(&out), + helmregistry.CacheOptRoot(filepath.Join(testCacheRootDir, helmregistry.CacheRootDir)), + ) + + assert.Nil(t, err, "failed creating cache") + + // init test client + registryClient, err := helmregistry.NewClient( + helmregistry.ClientOptDebug(true), + helmregistry.ClientOptWriter(&out), + helmregistry.ClientOptAuthorizer(&helmregistry.Authorizer{ + Client: client, + }), + helmregistry.ClientOptResolver(&helmregistry.Resolver{ + Resolver: resolver, + }), + helmregistry.ClientOptCache(cache), + ) + assert.Nil(t, err, "failed creating registry client") + + // create htpasswd file (w BCrypt, which is required) + pwBytes, err := bcrypt.GenerateFromPassword([]byte(testPassword), bcrypt.DefaultCost) + assert.Nil(t, err, "no error generating bcrypt password for test htpasswd file") + htpasswdPath := filepath.Join(testCacheRootDir, testHtpasswdFileBasename) + err = ioutil.WriteFile(htpasswdPath, []byte(fmt.Sprintf("%s:%s\n", testUsername, string(pwBytes))), 0644) + assert.Nil(t, err, "error creating test htpasswd file") + + // Registry config + config := &configuration.Configuration{} + port, err := freeport.GetFreePort() + assert.Nil(t, err, "no error finding free port for test registry") + dockerRegistryHost := fmt.Sprintf("localhost:%d", port) + config.HTTP.Addr = fmt.Sprintf(":%d", port) + config.HTTP.DrainTimeout = time.Duration(10) * time.Second + config.Storage = map[string]configuration.Parameters{"inmemory": map[string]interface{}{}} + config.Auth = configuration.Auth{ + "htpasswd": configuration.Parameters{ + "realm": "localhost", + "path": htpasswdPath, + }, + } + dockerRegistry, err := registry.NewRegistry(context.Background(), config) + assert.Nil(t, err, "no error creating test registry") + + // Start Docker registry + go dockerRegistry.ListenAndServe() + err = registryClient.Login(dockerRegistryHost, testUsername, testPassword, false) + assert.Nil(t, err, "failed to login to registry with username "+testUsername+" and password "+testPassword) + + ref, _ := helmregistry.ParseReference(fmt.Sprintf("%s/testrepo/testchart:1.2.3", dockerRegistryHost)) + ch, err := loader.LoadDir("testdata/local-subchart") + assert.Nil(t, err, "failed to load local chart") + err = registryClient.SaveChart(ch, ref) + assert.Nil(t, err, "failed to save chart") + err = registryClient.PushChart(ref) + assert.Nil(t, err, "failed to push chart") + + c := ChartDownloader{ + Out: os.Stderr, + Verify: VerifyIfPossible, + Keyring: "testdata/helm-test-key.pub", + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + Getters: append(getter.All(&cli.EnvSettings{ + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + }), helmregistry.NewRegistryGetterProvider(registryClient)), + Options: []getter.Option{ + getter.WithBasicAuth("username", "password"), + }, + } + // the filename becomes the {last segment of the image name}-{the image tag} + fname := "/testchart-1.2.3.tgz" + dest := ensure.TempDir(t) + version := "1.2.3" + where, _, err := c.DownloadTo(fmt.Sprintf("oci://%s/testrepo/testchart", dockerRegistryHost), version, dest) + if err != nil { + t.Fatal(err) + } + + if expect := filepath.Join(dest, fname); where != expect { + t.Errorf("Expected download to %s, got %s", expect, where) + } + + if _, err := os.Stat(filepath.Join(dest, fname)); err != nil { + t.Error(err) + } + + os.RemoveAll(testCacheRootDir) +} + +func TestDownloadToFromOCIRepositoryWithoutTagOrVersion(t *testing.T) { + var ( + CredentialsFileBasename = "config.json" + testCacheRootDir = "helm-registry-test" + ) + os.RemoveAll(testCacheRootDir) + os.Mkdir(testCacheRootDir, 0700) + + var out bytes.Buffer + credentialsFile := filepath.Join(testCacheRootDir, CredentialsFileBasename) + + client, err := auth.NewClient(credentialsFile) + assert.Nil(t, err, "no error creating auth client") + + resolver, err := client.Resolver(context.Background(), http.DefaultClient, false) + assert.Nil(t, err, "no error creating resolver") + + // create cache + cache, err := helmregistry.NewCache( + helmregistry.CacheOptDebug(true), + helmregistry.CacheOptWriter(&out), + helmregistry.CacheOptRoot(filepath.Join(testCacheRootDir, helmregistry.CacheRootDir)), + ) + + assert.Nil(t, err, "failed creating cache") + + // init test client + registryClient, err := helmregistry.NewClient( + helmregistry.ClientOptDebug(true), + helmregistry.ClientOptWriter(&out), + helmregistry.ClientOptAuthorizer(&helmregistry.Authorizer{ + Client: client, + }), + helmregistry.ClientOptResolver(&helmregistry.Resolver{ + Resolver: resolver, + }), + helmregistry.ClientOptCache(cache), + ) + + assert.Nil(t, err, "failed creating registry client") + + c := ChartDownloader{ + Out: os.Stderr, + Verify: VerifyIfPossible, + Keyring: "testdata/helm-test-key.pub", + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + Getters: append(getter.All(&cli.EnvSettings{ + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + }), helmregistry.NewRegistryGetterProvider(registryClient)), + Options: []getter.Option{ + getter.WithBasicAuth("username", "password"), + }, + } + // the filename becomes the {last segment of the image name}-{the image tag} + dest := ensure.TempDir(t) + _, _, err = c.DownloadTo("oci://testrepo/testchart", "", dest) + if err == nil { + t.Error("download succeeded without version or tag") + } + + os.RemoveAll(testCacheRootDir) +} + func TestScanReposForURL(t *testing.T) { c := ChartDownloader{ Out: os.Stderr, diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index 00198de0c..5b56357c9 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -31,6 +31,7 @@ import ( "github.com/pkg/errors" "sigs.k8s.io/yaml" + "helm.sh/helm/v3/internal/experimental/registry" "helm.sh/helm/v3/internal/resolver" "helm.sh/helm/v3/internal/third_party/dep/fs" "helm.sh/helm/v3/internal/urlutil" @@ -306,7 +307,7 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { }, } - if _, _, err := dl.DownloadTo(churl, "", destPath); err != nil { + if _, _, err := dl.DownloadTo(churl, dep.Version, destPath); err != nil { saveError = errors.Wrapf(err, "could not download %s", churl) break } @@ -400,6 +401,11 @@ Loop: continue } + // if repo is an OCI registry, continue + if strings.HasPrefix(dd.Repository, fmt.Sprintf("%s://", registry.OCIProtocol)) { + continue + } + if dd.Repository == "" { continue } @@ -545,7 +551,7 @@ func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error { // repoURL is the repository to search // // If it finds a URL that is "relative", it will prepend the repoURL. -func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (url, username, password string, err error) { +func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (chartURL, username, password string, err error) { for _, cr := range repos { if urlutil.Equal(repoURL, cr.Config.URL) { var entry repo.ChartVersions @@ -558,7 +564,7 @@ func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]* if err != nil { return } - url, err = normalizeURL(repoURL, ve.URLs[0]) + chartURL, err = normalizeURL(repoURL, ve.URLs[0]) if err != nil { return } @@ -567,12 +573,23 @@ func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]* return } } - url, err = repo.FindChartInRepoURL(repoURL, name, version, "", "", "", m.Getters) + + u, err := url.ParseRequestURI(repoURL) + if err != nil { + return + } + + if u.Scheme == registry.OCIProtocol { + chartURL = repoURL + return + } + + chartURL, err = repo.FindChartInRepoURL(repoURL, name, version, "", "", "", m.Getters) if err == nil { return } err = errors.Errorf("chart %s not found in %s", name, repoURL) - return + return chartURL, username, password, err } // findEntryByName finds an entry in the chart repository whose name matches the given name. diff --git a/pkg/downloader/manager_test.go b/pkg/downloader/manager_test.go index ea235c13f..8c533f879 100644 --- a/pkg/downloader/manager_test.go +++ b/pkg/downloader/manager_test.go @@ -96,6 +96,37 @@ func TestFindChartURL(t *testing.T) { } } +func TestFindChartUrlForOCIRepository(t *testing.T) { + var b bytes.Buffer + m := &Manager{ + Out: &b, + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + } + repos, err := m.loadChartRepositories() + if err != nil { + t.Fatal(err) + } + + name := "alpine" + version := "0.1.0" + repoURL := "oci://example.com/charts/alpine" + + churl, username, password, err := m.findChartURL(name, version, repoURL, repos) + if err != nil { + t.Fatal(err) + } + if churl != "oci://example.com/charts/alpine" { + t.Errorf("Unexpected URL %q", churl) + } + if username != "" { + t.Errorf("Unexpected username %q", username) + } + if password != "" { + t.Errorf("Unexpected password %q", password) + } +} + func TestGetRepoNames(t *testing.T) { b := bytes.NewBuffer(nil) m := &Manager{ diff --git a/pkg/repo/chartrepo.go b/pkg/repo/chartrepo.go index c2c366a1e..d6086bd36 100644 --- a/pkg/repo/chartrepo.go +++ b/pkg/repo/chartrepo.go @@ -205,7 +205,6 @@ func FindChartInRepoURL(repoURL, chartName, chartVersion, certFile, keyFile, caF // without adding repo to repositories, like FindChartInRepoURL, // but it also receives credentials for the chart repository. func FindChartInAuthRepoURL(repoURL, username, password, chartName, chartVersion, certFile, keyFile, caFile string, getters getter.Providers) (string, error) { - // Download and write the index file to a temporary location buf := make([]byte, 20) rand.Read(buf)