Merge pull request #8843 from bloodorangeio/oci-pull

Signed-off-by: Matt Farina <matt@mattfarina.com>
pull/9192/head
Matt Farina 4 years ago committed by GitHub
commit 49f895db6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -82,7 +82,7 @@ the contents of a chart.
This will produce an error if the chart cannot be loaded. This will produce an error if the chart cannot be loaded.
` `
func newDependencyCmd(out io.Writer) *cobra.Command { func newDependencyCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "dependency update|build|list", Use: "dependency update|build|list",
Aliases: []string{"dep", "dependencies"}, Aliases: []string{"dep", "dependencies"},
@ -92,7 +92,7 @@ func newDependencyCmd(out io.Writer) *cobra.Command {
} }
cmd.AddCommand(newDependencyListCmd(out)) cmd.AddCommand(newDependencyListCmd(out))
cmd.AddCommand(newDependencyUpdateCmd(out)) cmd.AddCommand(newDependencyUpdateCmd(cfg, out))
cmd.AddCommand(newDependencyBuildCmd(out)) cmd.AddCommand(newDependencyBuildCmd(out))
return cmd return cmd

@ -43,7 +43,7 @@ in the Chart.yaml file, but (b) at the wrong version.
` `
// newDependencyUpdateCmd creates a new dependency update command. // newDependencyUpdateCmd creates a new dependency update command.
func newDependencyUpdateCmd(out io.Writer) *cobra.Command { func newDependencyUpdateCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
client := action.NewDependency() client := action.NewDependency()
cmd := &cobra.Command{ cmd := &cobra.Command{
@ -63,6 +63,7 @@ func newDependencyUpdateCmd(out io.Writer) *cobra.Command {
Keyring: client.Keyring, Keyring: client.Keyring,
SkipUpdate: client.SkipRefresh, SkipUpdate: client.SkipRefresh,
Getters: getter.All(settings), Getters: getter.All(settings),
RegistryClient: cfg.RegistryClient,
RepositoryConfig: settings.RepositoryConfig, RepositoryConfig: settings.RepositoryConfig,
RepositoryCache: settings.RepositoryCache, RepositoryCache: settings.RepositoryCache,
Debug: settings.Debug, Debug: settings.Debug,

@ -40,6 +40,23 @@ func TestDependencyUpdateCmd(t *testing.T) {
defer srv.Stop() defer srv.Stop()
t.Logf("Listening on directory %s", srv.Root()) t.Logf("Listening on directory %s", srv.Root())
ociSrv, err := repotest.NewOCIServer(t, srv.Root())
if err != nil {
t.Fatal(err)
}
ociChartName := "oci-depending-chart"
c := createTestingMetadataForOCI(ociChartName, ociSrv.RegistryURL)
if err := chartutil.SaveDir(c, ociSrv.Dir); err != nil {
t.Fatal(err)
}
ociSrv.Run(t, repotest.WithDependingChart(c))
err = os.Setenv("HELM_EXPERIMENTAL_OCI", "1")
if err != nil {
t.Fatal("failed to set environment variable enabling OCI support")
}
if err := srv.LinkIndices(); err != nil { if err := srv.LinkIndices(); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -115,6 +132,22 @@ func TestDependencyUpdateCmd(t *testing.T) {
if _, err := os.Stat(unexpected); err == nil { if _, err := os.Stat(unexpected); err == nil {
t.Fatalf("Unexpected %q", unexpected) t.Fatalf("Unexpected %q", unexpected)
} }
// test for OCI charts
cmd := fmt.Sprintf("dependency update '%s' --repository-config %s --repository-cache %s --registry-config %s/config.json",
dir(ociChartName),
dir("repositories.yaml"),
dir(),
dir())
_, out, err = executeActionCommand(cmd)
if err != nil {
t.Logf("Output: %s", out)
t.Fatal(err)
}
expect = dir(ociChartName, "charts/oci-dependent-chart-0.1.0.tgz")
if _, err := os.Stat(expect); err != nil {
t.Fatal(err)
}
} }
func TestDependencyUpdateCmd_DontDeleteOldChartsOnError(t *testing.T) { func TestDependencyUpdateCmd_DontDeleteOldChartsOnError(t *testing.T) {
@ -193,6 +226,19 @@ func createTestingMetadata(name, baseURL string) *chart.Chart {
} }
} }
func createTestingMetadataForOCI(name, registryURL string) *chart.Chart {
return &chart.Chart{
Metadata: &chart.Metadata{
APIVersion: chart.APIVersionV2,
Name: name,
Version: "1.2.3",
Dependencies: []*chart.Dependency{
{Name: "oci-dependent-chart", Version: "0.1.0", Repository: fmt.Sprintf("oci://%s/u/ocitestuser", registryURL)},
},
},
}
}
// createTestingChart creates a basic chart that depends on reqtest-0.1.0 // createTestingChart creates a basic chart that depends on reqtest-0.1.0
// //
// The baseURL can be used to point to a particular repository server. // The baseURL can be used to point to a particular repository server.

@ -20,6 +20,7 @@ import (
"fmt" "fmt"
"io" "io"
"log" "log"
"strings"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@ -42,8 +43,8 @@ file, and MUST pass the verification process. Failure in any part of this will
result in an error, and the chart will not be saved locally. result in an error, and the chart will not be saved locally.
` `
func newPullCmd(out io.Writer) *cobra.Command { func newPullCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
client := action.NewPull() client := action.NewPullWithOpts(action.WithConfig(cfg))
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "pull [chart URL | repo/chartname] [...]", Use: "pull [chart URL | repo/chartname] [...]",
@ -64,6 +65,12 @@ func newPullCmd(out io.Writer) *cobra.Command {
client.Version = ">0.0.0-0" client.Version = ">0.0.0-0"
} }
if strings.HasPrefix(args[0], "oci://") {
if !FeatureGateOCI.IsEnabled() {
return FeatureGateOCI.Error()
}
}
for i := 0; i < len(args); i++ { for i := 0; i < len(args); i++ {
output, err := client.Run(args[i]) output, err := client.Run(args[i])
if err != nil { if err != nil {

@ -32,6 +32,13 @@ func TestPullCmd(t *testing.T) {
} }
defer srv.Stop() defer srv.Stop()
os.Setenv("HELM_EXPERIMENTAL_OCI", "1")
ociSrv, err := repotest.NewOCIServer(t, srv.Root())
if err != nil {
t.Fatal(err)
}
ociSrv.Run(t)
if err := srv.LinkIndices(); err != nil { if err := srv.LinkIndices(); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -139,23 +146,70 @@ func TestPullCmd(t *testing.T) {
failExpect: "Failed to fetch chart version", failExpect: "Failed to fetch chart version",
wantError: true, wantError: true,
}, },
{
name: "Fetch OCI Chart",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart --version 0.1.0", ociSrv.RegistryURL),
expectFile: "./oci-dependent-chart-0.1.0.tgz",
},
{
name: "Fetch OCI Chart with untar",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart --version 0.1.0 --untar", ociSrv.RegistryURL),
expectFile: "./oci-dependent-chart",
expectDir: true,
},
{
name: "Fetch OCI Chart with untar and untardir",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart --version 0.1.0 --untar --untardir ocitest2", ociSrv.RegistryURL),
expectFile: "./ocitest2",
expectDir: true,
},
{
name: "OCI Fetch untar when dir with same name existed",
args: fmt.Sprintf("oci-test-chart oci://%s/u/ocitestuser/oci-dependent-chart --version 0.1.0 --untar --untardir ocitest2 --untar --untardir ocitest2", ociSrv.RegistryURL),
wantError: true,
wantErrorMsg: fmt.Sprintf("failed to untar: a file or directory with the name %s already exists", filepath.Join(srv.Root(), "ocitest2")),
},
{
name: "Fail fetching non-existent OCI chart",
args: fmt.Sprintf("oci://%s/u/ocitestuser/nosuchthing --version 0.1.0", ociSrv.RegistryURL),
failExpect: "Failed to fetch",
wantError: true,
},
{
name: "Fail fetching OCI chart without version specified",
args: fmt.Sprintf("oci://%s/u/ocitestuser/nosuchthing", ociSrv.RegistryURL),
wantErrorMsg: "Error: --version flag is explicitly required for OCI registries",
wantError: true,
},
{
name: "Fail fetching OCI chart without version specified",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart:0.1.0", ociSrv.RegistryURL),
wantErrorMsg: "Error: --version flag is explicitly required for OCI registries",
wantError: true,
},
{
name: "Fail fetching OCI chart without version specified",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart:0.1.0 --version 0.1.0", ociSrv.RegistryURL),
wantError: true,
},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
outdir := srv.Root() outdir := srv.Root()
cmd := fmt.Sprintf("fetch %s -d '%s' --repository-config %s --repository-cache %s ", cmd := fmt.Sprintf("fetch %s -d '%s' --repository-config %s --repository-cache %s --registry-config %s",
tt.args, tt.args,
outdir, outdir,
filepath.Join(outdir, "repositories.yaml"), filepath.Join(outdir, "repositories.yaml"),
outdir, outdir,
filepath.Join(outdir, "config.json"),
) )
// Create file or Dir before helm pull --untar, see: https://github.com/helm/helm/issues/7182 // Create file or Dir before helm pull --untar, see: https://github.com/helm/helm/issues/7182
if tt.existFile != "" { if tt.existFile != "" {
file := filepath.Join(outdir, tt.existFile) file := filepath.Join(outdir, tt.existFile)
_, err := os.Create(file) _, err := os.Create(file)
if err != nil { if err != nil {
t.Fatal("err") t.Fatal(err)
} }
} }
if tt.existDir != "" { if tt.existDir != "" {

@ -153,12 +153,22 @@ func newRootCmd(actionConfig *action.Configuration, out io.Writer, args []string
flags.ParseErrorsWhitelist.UnknownFlags = true flags.ParseErrorsWhitelist.UnknownFlags = true
flags.Parse(args) flags.Parse(args)
registryClient, err := registry.NewClient(
registry.ClientOptDebug(settings.Debug),
registry.ClientOptWriter(out),
registry.ClientOptCredentialsFile(settings.RegistryConfig),
)
if err != nil {
return nil, err
}
actionConfig.RegistryClient = registryClient
// Add subcommands // Add subcommands
cmd.AddCommand( cmd.AddCommand(
// chart commands // chart commands
newCreateCmd(out), newCreateCmd(out),
newDependencyCmd(out), newDependencyCmd(actionConfig, out),
newPullCmd(out), newPullCmd(actionConfig, out),
newShowCmd(out), newShowCmd(out),
newLintCmd(out), newLintCmd(out),
newPackageCmd(out), newPackageCmd(out),
@ -188,15 +198,6 @@ func newRootCmd(actionConfig *action.Configuration, out io.Writer, args []string
) )
// Add *experimental* subcommands // Add *experimental* subcommands
registryClient, err := registry.NewClient(
registry.ClientOptDebug(settings.Debug),
registry.ClientOptWriter(out),
registry.ClientOptCredentialsFile(settings.RegistryConfig),
)
if err != nil {
return nil, err
}
actionConfig.RegistryClient = registryClient
cmd.AddCommand( cmd.AddCommand(
newRegistryCmd(actionConfig, out), newRegistryCmd(actionConfig, out),
newChartCmd(actionConfig, out), newChartCmd(actionConfig, out),

@ -27,6 +27,7 @@ require (
github.com/mitchellh/copystructure v1.0.0 github.com/mitchellh/copystructure v1.0.0
github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.1 github.com/opencontainers/image-spec v1.0.1
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351 github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351
github.com/sirupsen/logrus v1.7.0 github.com/sirupsen/logrus v1.7.0

@ -17,6 +17,7 @@ limitations under the License.
package registry // import "helm.sh/helm/v3/internal/experimental/registry" package registry // import "helm.sh/helm/v3/internal/experimental/registry"
import ( import (
"bytes"
"context" "context"
"fmt" "fmt"
"io" "io"
@ -25,6 +26,7 @@ import (
"sort" "sort"
auth "github.com/deislabs/oras/pkg/auth/docker" auth "github.com/deislabs/oras/pkg/auth/docker"
"github.com/deislabs/oras/pkg/content"
"github.com/deislabs/oras/pkg/oras" "github.com/deislabs/oras/pkg/oras"
"github.com/gosuri/uitable" "github.com/gosuri/uitable"
ocispec "github.com/opencontainers/image-spec/specs-go/v1" ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -144,7 +146,60 @@ func (c *Client) PushChart(ref *Reference) error {
} }
// PullChart downloads a chart from a registry // PullChart downloads a chart from a registry
func (c *Client) PullChart(ref *Reference) error { func (c *Client) PullChart(ref *Reference) (*bytes.Buffer, error) {
buf := bytes.NewBuffer(nil)
if ref.Tag == "" {
return buf, errors.New("tag explicitly required")
}
fmt.Fprintf(c.out, "%s: Pulling from %s\n", ref.Tag, ref.Repo)
store := content.NewMemoryStore()
fullname := ref.FullName()
_ = fullname
_, layerDescriptors, err := oras.Pull(ctx(c.out, c.debug), c.resolver, ref.FullName(), store,
oras.WithPullEmptyNameAllowed(),
oras.WithAllowedMediaTypes(KnownMediaTypes()))
if err != nil {
return buf, err
}
numLayers := len(layerDescriptors)
if numLayers < 1 {
return buf, errors.New(
fmt.Sprintf("manifest does not contain at least 1 layer (total: %d)", numLayers))
}
var contentLayer *ocispec.Descriptor
for _, layer := range layerDescriptors {
layer := layer
switch layer.MediaType {
case HelmChartContentLayerMediaType:
contentLayer = &layer
}
}
if contentLayer == nil {
return buf, errors.New(
fmt.Sprintf("manifest does not contain a layer with mediatype %s",
HelmChartContentLayerMediaType))
}
_, b, ok := store.Get(*contentLayer)
if !ok {
return buf, errors.Errorf("Unable to retrieve blob with digest %s", contentLayer.Digest)
}
buf = bytes.NewBuffer(b)
return buf, nil
}
// PullChartToCache pulls a chart from an OCI Registry to the Registry Cache.
// This function is needed for `helm chart pull`, which is experimental and will be deprecated soon.
// Likewise, the Registry cache will soon be deprecated as will this function.
func (c *Client) PullChartToCache(ref *Reference) error {
if ref.Tag == "" { if ref.Tag == "" {
return errors.New("tag explicitly required") return errors.New("tag explicitly required")
} }

@ -22,7 +22,6 @@ import (
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"net"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url" "net/url"
@ -33,12 +32,12 @@ import (
"time" "time"
"github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/errdefs"
auth "github.com/deislabs/oras/pkg/auth/docker" auth "github.com/deislabs/oras/pkg/auth/docker"
"github.com/docker/distribution/configuration" "github.com/docker/distribution/configuration"
"github.com/docker/distribution/registry" "github.com/docker/distribution/registry"
_ "github.com/docker/distribution/registry/auth/htpasswd" _ "github.com/docker/distribution/registry/auth/htpasswd"
_ "github.com/docker/distribution/registry/storage/driver/inmemory" _ "github.com/docker/distribution/registry/storage/driver/inmemory"
"github.com/phayes/freeport"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"golang.org/x/crypto/bcrypt" "golang.org/x/crypto/bcrypt"
@ -107,7 +106,7 @@ func (suite *RegistryClientTestSuite) SetupSuite() {
// Registry config // Registry config
config := &configuration.Configuration{} config := &configuration.Configuration{}
port, err := getFreePort() port, err := freeport.GetFreePort()
suite.Nil(err, "no error finding free port for test registry") suite.Nil(err, "no error finding free port for test registry")
suite.DockerRegistryHost = fmt.Sprintf("localhost:%d", port) suite.DockerRegistryHost = fmt.Sprintf("localhost:%d", port)
config.HTTP.Addr = fmt.Sprintf(":%d", port) config.HTTP.Addr = fmt.Sprintf(":%d", port)
@ -202,13 +201,13 @@ func (suite *RegistryClientTestSuite) Test_4_PullChart() {
// non-existent ref // non-existent ref
ref, err := ParseReference(fmt.Sprintf("%s/testrepo/whodis:9.9.9", suite.DockerRegistryHost)) ref, err := ParseReference(fmt.Sprintf("%s/testrepo/whodis:9.9.9", suite.DockerRegistryHost))
suite.Nil(err) suite.Nil(err)
err = suite.RegistryClient.PullChart(ref) _, err = suite.RegistryClient.PullChart(ref)
suite.NotNil(err) suite.NotNil(err)
// existing ref // existing ref
ref, err = ParseReference(fmt.Sprintf("%s/testrepo/testchart:1.2.3", suite.DockerRegistryHost)) ref, err = ParseReference(fmt.Sprintf("%s/testrepo/testchart:1.2.3", suite.DockerRegistryHost))
suite.Nil(err) suite.Nil(err)
err = suite.RegistryClient.PullChart(ref) _, err = suite.RegistryClient.PullChart(ref)
suite.Nil(err) suite.Nil(err)
} }
@ -245,7 +244,7 @@ func (suite *RegistryClientTestSuite) Test_8_ManInTheMiddle() {
suite.Nil(err) suite.Nil(err)
// returns content that does not match the expected digest // returns content that does not match the expected digest
err = suite.RegistryClient.PullChart(ref) _, err = suite.RegistryClient.PullChart(ref)
suite.NotNil(err) suite.NotNil(err)
suite.True(errdefs.IsFailedPrecondition(err)) suite.True(errdefs.IsFailedPrecondition(err))
} }
@ -254,21 +253,6 @@ func TestRegistryClientTestSuite(t *testing.T) {
suite.Run(t, new(RegistryClientTestSuite)) suite.Run(t, new(RegistryClientTestSuite))
} }
// borrowed from https://github.com/phayes/freeport
func getFreePort() (int, error) {
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
if err != nil {
return 0, err
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return 0, err
}
defer l.Close()
return l.Addr().(*net.TCPAddr).Port, nil
}
func initCompromisedRegistryTestServer() string { func initCompromisedRegistryTestServer() string {
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.Contains(r.URL.Path, "manifests") { if strings.Contains(r.URL.Path, "manifests") {

@ -28,11 +28,14 @@ import (
"helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/gates"
"helm.sh/helm/v3/pkg/helmpath" "helm.sh/helm/v3/pkg/helmpath"
"helm.sh/helm/v3/pkg/provenance" "helm.sh/helm/v3/pkg/provenance"
"helm.sh/helm/v3/pkg/repo" "helm.sh/helm/v3/pkg/repo"
) )
const FeatureGateOCI = gates.Gate("HELM_EXPERIMENTAL_OCI")
// Resolver resolves dependencies from semantic version ranges to a particular version. // Resolver resolves dependencies from semantic version ranges to a particular version.
type Resolver struct { type Resolver struct {
chartpath string chartpath string
@ -88,6 +91,7 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string
} }
continue continue
} }
constraint, err := semver.NewConstraint(d.Version) constraint, err := semver.NewConstraint(d.Version)
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "dependency %q has an invalid version/constraint format", d.Name) return nil, errors.Wrapf(err, "dependency %q has an invalid version/constraint format", d.Name)
@ -104,21 +108,34 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string
continue continue
} }
var vs repo.ChartVersions
var version string
var ok bool
found := true
if !strings.HasPrefix(d.Repository, "oci://") {
repoIndex, err := repo.LoadIndexFile(filepath.Join(r.cachepath, helmpath.CacheIndexFile(repoName))) repoIndex, err := repo.LoadIndexFile(filepath.Join(r.cachepath, helmpath.CacheIndexFile(repoName)))
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "no cached repository for %s found. (try 'helm repo update')", repoName) return nil, errors.Wrapf(err, "no cached repository for %s found. (try 'helm repo update')", repoName)
} }
vs, ok := repoIndex.Entries[d.Name] vs, ok = repoIndex.Entries[d.Name]
if !ok { if !ok {
return nil, errors.Errorf("%s chart not found in repo %s", d.Name, d.Repository) return nil, errors.Errorf("%s chart not found in repo %s", d.Name, d.Repository)
} }
found = false
} else {
version = d.Version
if !FeatureGateOCI.IsEnabled() {
return nil, errors.Wrapf(FeatureGateOCI.Error(),
"repository %s is an OCI registry", d.Repository)
}
}
locked[i] = &chart.Dependency{ locked[i] = &chart.Dependency{
Name: d.Name, Name: d.Name,
Repository: d.Repository, Repository: d.Repository,
Version: version,
} }
found := false
// The version are already sorted and hence the first one to satisfy the constraint is used // The version are already sorted and hence the first one to satisfy the constraint is used
for _, ver := range vs { for _, ver := range vs {
v, err := semver.NewVersion(ver.Version) v, err := semver.NewVersion(ver.Version)

@ -40,5 +40,5 @@ func (a *ChartPull) Run(out io.Writer, ref string) error {
if err != nil { if err != nil {
return err return err
} }
return a.cfg.RegistryClient.PullChart(r) return a.cfg.RegistryClient.PullChartToCache(r)
} }

@ -45,11 +45,30 @@ type Pull struct {
VerifyLater bool VerifyLater bool
UntarDir string UntarDir string
DestDir string DestDir string
cfg *Configuration
} }
// NewPull creates a new Pull object with the given configuration. type PullOpt func(*Pull)
func WithConfig(cfg *Configuration) PullOpt {
return func(p *Pull) {
p.cfg = cfg
}
}
// NewPull creates a new Pull object.
func NewPull() *Pull { func NewPull() *Pull {
return &Pull{} return NewPullWithOpts()
}
// NewPull creates a new pull, with configuration options.
func NewPullWithOpts(opts ...PullOpt) *Pull {
p := &Pull{}
for _, fn := range opts {
fn(p)
}
return p
} }
// Run executes 'helm pull' against the given release. // Run executes 'helm pull' against the given release.
@ -70,6 +89,16 @@ func (p *Pull) Run(chartRef string) (string, error) {
RepositoryCache: p.Settings.RepositoryCache, RepositoryCache: p.Settings.RepositoryCache,
} }
if strings.HasPrefix(chartRef, "oci://") {
if p.Version == "" {
return out.String(), errors.Errorf("--version flag is explicitly required for OCI registries")
}
c.Options = append(c.Options,
getter.WithRegistryClient(p.cfg.RegistryClient),
getter.WithTagName(p.Version))
}
if p.Verify { if p.Verify {
c.Verify = downloader.VerifyAlways c.Verify = downloader.VerifyAlways
} else if p.VerifyLater { } else if p.VerifyLater {
@ -123,6 +152,7 @@ func (p *Pull) Run(chartRef string) (string, error) {
_, chartName := filepath.Split(chartRef) _, chartName := filepath.Split(chartRef)
udCheck = filepath.Join(udCheck, chartName) udCheck = filepath.Join(udCheck, chartName)
} }
if _, err := os.Stat(udCheck); err != nil { if _, err := os.Stat(udCheck); err != nil {
if err := os.MkdirAll(udCheck, 0755); err != nil { if err := os.MkdirAll(udCheck, 0755); err != nil {
return out.String(), errors.Wrap(err, "failed to untar (mkdir)") return out.String(), errors.Wrap(err, "failed to untar (mkdir)")

@ -25,6 +25,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"helm.sh/helm/v3/internal/experimental/registry"
"helm.sh/helm/v3/internal/fileutil" "helm.sh/helm/v3/internal/fileutil"
"helm.sh/helm/v3/internal/urlutil" "helm.sh/helm/v3/internal/urlutil"
"helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/getter"
@ -68,6 +69,7 @@ type ChartDownloader struct {
Getters getter.Providers Getters getter.Providers
// Options provide parameters to be passed along to the Getter being initialized. // Options provide parameters to be passed along to the Getter being initialized.
Options []getter.Option Options []getter.Option
RegistryClient *registry.Client
RepositoryConfig string RepositoryConfig string
RepositoryCache string RepositoryCache string
} }
@ -100,6 +102,10 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven
} }
name := filepath.Base(u.Path) name := filepath.Base(u.Path)
if u.Scheme == "oci" {
name = fmt.Sprintf("%s-%s.tgz", name, version)
}
destfile := filepath.Join(dest, name) destfile := filepath.Join(dest, name)
if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil { if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil {
return destfile, nil, err return destfile, nil, err

@ -26,6 +26,7 @@ import (
"os" "os"
"path" "path"
"path/filepath" "path/filepath"
"regexp"
"strings" "strings"
"sync" "sync"
@ -33,6 +34,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"sigs.k8s.io/yaml" "sigs.k8s.io/yaml"
"helm.sh/helm/v3/internal/experimental/registry"
"helm.sh/helm/v3/internal/resolver" "helm.sh/helm/v3/internal/resolver"
"helm.sh/helm/v3/internal/third_party/dep/fs" "helm.sh/helm/v3/internal/third_party/dep/fs"
"helm.sh/helm/v3/internal/urlutil" "helm.sh/helm/v3/internal/urlutil"
@ -71,6 +73,7 @@ type Manager struct {
SkipUpdate bool SkipUpdate bool
// Getter collection for the operation // Getter collection for the operation
Getters []getter.Provider Getters []getter.Provider
RegistryClient *registry.Client
RepositoryConfig string RepositoryConfig string
RepositoryCache string RepositoryCache string
} }
@ -332,7 +335,24 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error {
}, },
} }
if _, _, err := dl.DownloadTo(churl, "", destPath); err != nil { version := ""
if strings.HasPrefix(churl, "oci://") {
if !resolver.FeatureGateOCI.IsEnabled() {
return errors.Wrapf(resolver.FeatureGateOCI.Error(),
"the repository %s is an OCI registry", churl)
}
churl, version, err = parseOCIRef(churl)
if err != nil {
return errors.Wrapf(err, "could not parse OCI reference")
}
dl.Options = append(dl.Options,
getter.WithRegistryClient(m.RegistryClient),
getter.WithTagName(version))
}
_, _, err = dl.DownloadTo(churl, version, destPath)
if err != nil {
saveError = errors.Wrapf(err, "could not download %s", churl) saveError = errors.Wrapf(err, "could not download %s", churl)
break break
} }
@ -375,6 +395,18 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error {
return nil return nil
} }
func parseOCIRef(chartRef string) (string, string, error) {
refTagRegexp := regexp.MustCompile(`^(oci://[^:]+(:[0-9]{1,5})?[^:]+):(.*)$`)
caps := refTagRegexp.FindStringSubmatch(chartRef)
if len(caps) != 4 {
return "", "", errors.Errorf("improperly formatted oci chart reference: %s", chartRef)
}
chartRef = caps[1]
tag := caps[3]
return chartRef, tag, nil
}
// safeDeleteDep deletes any versions of the given dependency in the given directory. // safeDeleteDep deletes any versions of the given dependency in the given directory.
// //
// It does this by first matching the file name to an expected pattern, then loading // It does this by first matching the file name to an expected pattern, then loading
@ -539,6 +571,11 @@ func (m *Manager) resolveRepoNames(deps []*chart.Dependency) (map[string]string,
continue continue
} }
if strings.HasPrefix(dd.Repository, "oci://") {
reposMap[dd.Name] = dd.Repository
continue
}
found := false found := false
for _, repo := range repos { for _, repo := range repos {
@ -648,7 +685,12 @@ func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error {
// //
// If it finds a URL that is "relative", it will prepend the repoURL. // If it finds a URL that is "relative", it will prepend the repoURL.
func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (url, username, password string, err error) { func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (url, username, password string, err error) {
if strings.HasPrefix(repoURL, "oci://") {
return fmt.Sprintf("%s/%s:%s", repoURL, name, version), "", "", nil
}
for _, cr := range repos { for _, cr := range repos {
if urlutil.Equal(repoURL, cr.Config.URL) { if urlutil.Equal(repoURL, cr.Config.URL) {
var entry repo.ChartVersions var entry repo.ChartVersions
entry, err = findEntryByName(name, cr) entry, err = findEntryByName(name, cr)
@ -671,10 +713,10 @@ func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*
} }
url, err = repo.FindChartInRepoURL(repoURL, name, version, "", "", "", m.Getters) url, err = repo.FindChartInRepoURL(repoURL, name, version, "", "", "", m.Getters)
if err == nil { if err == nil {
return return url, username, password, err
} }
err = errors.Errorf("chart %s not found in %s: %s", name, repoURL, err) err = errors.Errorf("chart %s not found in %s: %s", name, repoURL, err)
return return url, username, password, err
} }
// findEntryByName finds an entry in the chart repository whose name matches the given name. // findEntryByName finds an entry in the chart repository whose name matches the given name.

@ -22,6 +22,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"helm.sh/helm/v3/internal/experimental/registry"
"helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/cli"
) )
@ -33,10 +34,13 @@ type options struct {
certFile string certFile string
keyFile string keyFile string
caFile string caFile string
unTar bool
insecureSkipVerifyTLS bool insecureSkipVerifyTLS bool
username string username string
password string password string
userAgent string userAgent string
version string
registryClient *registry.Client
timeout time.Duration timeout time.Duration
} }
@ -90,6 +94,24 @@ func WithTimeout(timeout time.Duration) Option {
} }
} }
func WithTagName(tagname string) Option {
return func(opts *options) {
opts.version = tagname
}
}
func WithRegistryClient(client *registry.Client) Option {
return func(opts *options) {
opts.registryClient = client
}
}
func WithUntar() Option {
return func(opts *options) {
opts.unTar = true
}
}
// Getter is an interface to support GET to the specified URL. // Getter is an interface to support GET to the specified URL.
type Getter interface { type Getter interface {
// Get file content by url string // Get file content by url string
@ -139,11 +161,16 @@ var httpProvider = Provider{
New: NewHTTPGetter, New: NewHTTPGetter,
} }
var ociProvider = Provider{
Schemes: []string{"oci"},
New: NewOCIGetter,
}
// All finds all of the registered getters as a list of Provider instances. // All finds all of the registered getters as a list of Provider instances.
// Currently, the built-in getters and the discovered plugins with downloader // Currently, the built-in getters and the discovered plugins with downloader
// notations are collected. // notations are collected.
func All(settings *cli.EnvSettings) Providers { func All(settings *cli.EnvSettings) Providers {
result := Providers{httpProvider} result := Providers{httpProvider, ociProvider}
pluginDownloaders, _ := collectPlugins(settings) pluginDownloaders, _ := collectPlugins(settings)
result = append(result, pluginDownloaders...) result = append(result, pluginDownloaders...)
return result return result

@ -57,8 +57,8 @@ func TestAll(t *testing.T) {
env.PluginsDirectory = pluginDir env.PluginsDirectory = pluginDir
all := All(env) all := All(env)
if len(all) != 3 { if len(all) != 4 {
t.Errorf("expected 3 providers (default plus two plugins), got %d", len(all)) t.Errorf("expected 4 providers (default plus three plugins), got %d", len(all))
} }
if _, err := all.ByScheme("test2"); err != nil { if _, err := all.ByScheme("test2"); err != nil {

@ -0,0 +1,69 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package getter
import (
"bytes"
"fmt"
"strings"
"helm.sh/helm/v3/internal/experimental/registry"
)
// OCIGetter is the default HTTP(/S) backend handler
type OCIGetter struct {
opts options
}
//Get performs a Get from repo.Getter and returns the body.
func (g *OCIGetter) Get(href string, options ...Option) (*bytes.Buffer, error) {
for _, opt := range options {
opt(&g.opts)
}
return g.get(href)
}
func (g *OCIGetter) get(href string) (*bytes.Buffer, error) {
client := g.opts.registryClient
ref := strings.TrimPrefix(href, "oci://")
if version := g.opts.version; version != "" {
ref = fmt.Sprintf("%s:%s", ref, version)
}
r, err := registry.ParseReference(ref)
if err != nil {
return nil, err
}
buf, err := client.PullChart(r)
if err != nil {
return nil, err
}
return buf, nil
}
// NewOCIGetter constructs a valid http/https client as a Getter
func NewOCIGetter(options ...Option) (Getter, error) {
var client OCIGetter
for _, opt := range options {
opt(&client.opts)
}
return &client, nil
}

@ -16,17 +16,30 @@ limitations under the License.
package repotest package repotest
import ( import (
"context"
"fmt"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
"time"
"helm.sh/helm/v3/internal/tlsutil" auth "github.com/deislabs/oras/pkg/auth/docker"
"github.com/docker/distribution/configuration"
"github.com/docker/distribution/registry"
_ "github.com/docker/distribution/registry/auth/htpasswd" // used for docker test registry
_ "github.com/docker/distribution/registry/storage/driver/inmemory" // used for docker test registry
"github.com/phayes/freeport"
"golang.org/x/crypto/bcrypt"
"sigs.k8s.io/yaml" "sigs.k8s.io/yaml"
ociRegistry "helm.sh/helm/v3/internal/experimental/registry"
"helm.sh/helm/v3/internal/tlsutil"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
"helm.sh/helm/v3/pkg/repo" "helm.sh/helm/v3/pkg/repo"
) )
@ -43,6 +56,166 @@ func NewTempServerWithCleanup(t *testing.T, glob string) (*Server, error) {
return srv, err return srv, err
} }
type OCIServer struct {
*registry.Registry
RegistryURL string
Dir string
TestUsername string
TestPassword string
Client *ociRegistry.Client
}
type OCIServerRunConfig struct {
DependingChart *chart.Chart
}
type OCIServerOpt func(config *OCIServerRunConfig)
func WithDependingChart(c *chart.Chart) OCIServerOpt {
return func(config *OCIServerRunConfig) {
config.DependingChart = c
}
}
func NewOCIServer(t *testing.T, dir string) (*OCIServer, error) {
testHtpasswdFileBasename := "authtest.htpasswd"
testUsername, testPassword := "username", "password"
pwBytes, err := bcrypt.GenerateFromPassword([]byte(testPassword), bcrypt.DefaultCost)
if err != nil {
t.Fatal("error generating bcrypt password for test htpasswd file")
}
htpasswdPath := filepath.Join(dir, testHtpasswdFileBasename)
err = ioutil.WriteFile(htpasswdPath, []byte(fmt.Sprintf("%s:%s\n", testUsername, string(pwBytes))), 0644)
if err != nil {
t.Fatalf("error creating test htpasswd file")
}
// Registry config
config := &configuration.Configuration{}
port, err := freeport.GetFreePort()
if err != nil {
t.Fatalf("error finding free port for test registry")
}
config.HTTP.Addr = fmt.Sprintf(":%d", port)
config.HTTP.DrainTimeout = time.Duration(10) * time.Second
config.Storage = map[string]configuration.Parameters{"inmemory": map[string]interface{}{}}
config.Auth = configuration.Auth{
"htpasswd": configuration.Parameters{
"realm": "localhost",
"path": htpasswdPath,
},
}
registryURL := fmt.Sprintf("localhost:%d", port)
r, err := registry.NewRegistry(context.Background(), config)
if err != nil {
t.Fatal(err)
}
return &OCIServer{
Registry: r,
RegistryURL: registryURL,
TestUsername: testUsername,
TestPassword: testPassword,
Dir: dir,
}, nil
}
func (srv *OCIServer) Run(t *testing.T, opts ...OCIServerOpt) {
cfg := &OCIServerRunConfig{}
for _, fn := range opts {
fn(cfg)
}
go srv.ListenAndServe()
credentialsFile := filepath.Join(srv.Dir, "config.json")
client, err := auth.NewClient(credentialsFile)
if err != nil {
t.Fatalf("error creating auth client")
}
resolver, err := client.Resolver(context.Background(), http.DefaultClient, false)
if err != nil {
t.Fatalf("error creating resolver")
}
// init test client
registryClient, err := ociRegistry.NewClient(
ociRegistry.ClientOptDebug(true),
ociRegistry.ClientOptWriter(os.Stdout),
ociRegistry.ClientOptAuthorizer(&ociRegistry.Authorizer{
Client: client,
}),
ociRegistry.ClientOptResolver(&ociRegistry.Resolver{
Resolver: resolver,
}),
)
if err != nil {
t.Fatalf("error creating registry client")
}
err = registryClient.Login(srv.RegistryURL, srv.TestUsername, srv.TestPassword, false)
if err != nil {
t.Fatalf("error logging into registry with good credentials")
}
ref, err := ociRegistry.ParseReference(fmt.Sprintf("%s/u/ocitestuser/oci-dependent-chart:0.1.0", srv.RegistryURL))
if err != nil {
t.Fatalf("")
}
err = chartutil.ExpandFile(srv.Dir, filepath.Join(srv.Dir, "oci-dependent-chart-0.1.0.tgz"))
if err != nil {
t.Fatal(err)
}
// valid chart
ch, err := loader.LoadDir(filepath.Join(srv.Dir, "oci-dependent-chart"))
if err != nil {
t.Fatal("error loading chart")
}
err = os.RemoveAll(filepath.Join(srv.Dir, "oci-dependent-chart"))
if err != nil {
t.Fatal("error removing chart before push")
}
err = registryClient.SaveChart(ch, ref)
if err != nil {
t.Fatal("error saving chart")
}
err = registryClient.PushChart(ref)
if err != nil {
t.Fatal("error pushing chart")
}
if cfg.DependingChart != nil {
c := cfg.DependingChart
dependingRef, err := ociRegistry.ParseReference(fmt.Sprintf("%s/u/ocitestuser/oci-depending-chart:1.2.3", srv.RegistryURL))
if err != nil {
t.Fatal("error parsing reference for depending chart reference")
}
err = registryClient.SaveChart(c, dependingRef)
if err != nil {
t.Fatal("error saving depending chart")
}
err = registryClient.PushChart(dependingRef)
if err != nil {
t.Fatal("error pushing depending chart")
}
}
srv.Client = registryClient
}
// NewTempServer creates a server inside of a temp dir. // NewTempServer creates a server inside of a temp dir.
// //
// If the passed in string is not "", it will be treated as a shell glob, and files // If the passed in string is not "", it will be treated as a shell glob, and files

Loading…
Cancel
Save