Merge pull request #10527 from scottrigby/oci-deppendency-version-range-support

OCI version range support

go.mod

@@ -42,6 +42,6 @@ require (
 	k8s.io/client-go v0.23.1
 	k8s.io/klog/v2 v2.30.0
 	k8s.io/kubectl v0.23.1
-	oras.land/oras-go v1.1.0-rc3
+	oras.land/oras-go v1.1.0
 	sigs.k8s.io/yaml v1.3.0
 )

go.sum

@@ -1746,8 +1746,8 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-oras.land/oras-go v1.1.0-rc3 h1:+HdHR0Lgm/0jmjF4SqUif8Ky2XNWhrcP4hxGVvtKIUI=
+oras.land/oras-go v1.1.0 h1:tfWM1RT7PzUwWphqHU6ptPU3ZhwVnSw/9nEGf519rYg=
-oras.land/oras-go v1.1.0-rc3/go.mod h1:1A7vR/0KknT2UkJVWh+xMi95I/AhK8ZrxrnUSmXN0bQ=
+oras.land/oras-go v1.1.0/go.mod h1:1A7vR/0KknT2UkJVWh+xMi95I/AhK8ZrxrnUSmXN0bQ=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

internal/experimental/registry/client.go

@@ -17,13 +17,16 @@ limitations under the License.
 package registry // import "helm.sh/helm/v3/internal/experimental/registry"
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"sort"
 	"strings"
 
+	"github.com/Masterminds/semver/v3"
 	"github.com/containerd/containerd/remotes"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -31,6 +34,9 @@ import (
 	dockerauth "oras.land/oras-go/pkg/auth/docker"
 	"oras.land/oras-go/pkg/content"
 	"oras.land/oras-go/pkg/oras"
+	"oras.land/oras-go/pkg/registry"
+	registryremote "oras.land/oras-go/pkg/registry/remote"
+	registryauth "oras.land/oras-go/pkg/registry/remote/auth"
 
 	"helm.sh/helm/v3/internal/version"
 	"helm.sh/helm/v3/pkg/chart"
@@ -49,10 +55,11 @@ type (
 	Client struct {
 		debug bool
 		// path to repository config file e.g. ~/.docker/config.json
-		credentialsFile string
+		credentialsFile    string
-		out             io.Writer
+		out                io.Writer
-		authorizer      auth.Client
+		authorizer         auth.Client
-		resolver        remotes.Resolver
+		registryAuthorizer *registryauth.Client
+		resolver           remotes.Resolver
 	}
 
 	// ClientOption allows specifying various settings configurable by the user for overriding the defaults
@@ -88,6 +95,32 @@ func NewClient(options ...ClientOption) (*Client, error) {
 		}
 		client.resolver = resolver
 	}
+	if client.registryAuthorizer == nil {
+		client.registryAuthorizer = &registryauth.Client{
+			Header: http.Header{
+				"User-Agent": {version.GetUserAgent()},
+			},
+			Cache: registryauth.DefaultCache,
+			Credential: func(ctx context.Context, reg string) (registryauth.Credential, error) {
+				dockerClient, ok := client.authorizer.(*dockerauth.Client)
+				if !ok {
+					return registryauth.EmptyCredential, errors.New("unable to obtain docker client")
+				}
+
+				username, password, err := dockerClient.Credential(reg)
+				if err != nil {
+					return registryauth.EmptyCredential, errors.New("unable to retrieve credentials")
+				}
+
+				return registryauth.Credential{
+					Username: username,
+					Password: password,
+				}, nil
+
+			},
+		}
+
+	}
 	return client, nil
 }
 
@@ -539,3 +572,55 @@ func PushOptStrictMode(strictMode bool) PushOption {
 		operation.strictMode = strictMode
 	}
 }
+
+// Tags provides a sorted list all semver compliant tags for a given repository
+func (c *Client) Tags(ref string) ([]string, error) {
+	parsedReference, err := registry.ParseReference(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	repository := registryremote.Repository{
+		Reference: parsedReference,
+		Client:    c.registryAuthorizer,
+	}
+
+	var registryTags []string
+
+	for {
+		registryTags, err = registry.Tags(ctx(c.out, c.debug), &repository)
+		if err != nil {
+			// Fallback to http based request
+			if !repository.PlainHTTP && strings.Contains(err.Error(), "server gave HTTP response") {
+				repository.PlainHTTP = true
+				continue
+			}
+			return nil, err
+		}
+
+		break
+
+	}
+
+	var tagVersions []*semver.Version
+	for _, tag := range registryTags {
+		// Change underscore (_) back to plus (+) for Helm
+		// See https://github.com/helm/helm/issues/10166
+		tagVersion, err := semver.StrictNewVersion(strings.ReplaceAll(tag, "_", "+"))
+		if err == nil {
+			tagVersions = append(tagVersions, tagVersion)
+		}
+	}
+
+	// Sort the collection
+	sort.Sort(sort.Reverse(semver.Collection(tagVersions)))
+
+	tags := make([]string, len(tagVersions))
+
+	for iTv, tv := range tagVersions {
+		tags[iTv] = tv.String()
+	}
+
+	return tags, nil
+
+}

internal/experimental/registry/client_test.go

@@ -294,7 +294,23 @@ func (suite *RegistryClientTestSuite) Test_2_Pull() {
 	suite.Equal(provData, result.Prov.Data)
 }
 
-func (suite *RegistryClientTestSuite) Test_3_Logout() {
+func (suite *RegistryClientTestSuite) Test_3_Tags() {
+
+	// Load test chart (to build ref pushed in previous test)
+	chartData, err := ioutil.ReadFile("../../../pkg/downloader/testdata/local-subchart-0.1.0.tgz")
+	suite.Nil(err, "no error loading test chart")
+	meta, err := extractChartMeta(chartData)
+	suite.Nil(err, "no error extracting chart meta")
+	ref := fmt.Sprintf("%s/testrepo/%s", suite.DockerRegistryHost, meta.Name)
+
+	// Query for tags and validate length
+	tags, err := suite.RegistryClient.Tags(ref)
+	suite.Nil(err, "no error retrieving tags")
+	suite.Equal(1, len(tags))
+
+}
+
+func (suite *RegistryClientTestSuite) Test_4_Logout() {
 	err := suite.RegistryClient.Logout("this-host-aint-real:5000")
 	suite.NotNil(err, "error logging out of registry that has no entry")
 
@@ -302,7 +318,7 @@ func (suite *RegistryClientTestSuite) Test_3_Logout() {
 	suite.Nil(err, "no error logging out of registry")
 }
 
-func (suite *RegistryClientTestSuite) Test_4_ManInTheMiddle() {
+func (suite *RegistryClientTestSuite) Test_5_ManInTheMiddle() {
 	ref := fmt.Sprintf("%s/testrepo/supposedlysafechart:9.9.9", suite.CompromisedRegistryHost)
 
 	// returns content that does not match the expected digest

internal/experimental/registry/util.go

@@ -23,6 +23,8 @@ import (
 	"io"
 	"strings"
 
+	"github.com/Masterminds/semver/v3"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	orascontext "oras.land/oras-go/pkg/context"
 	"oras.land/oras-go/pkg/registry"
@@ -36,6 +38,53 @@ func IsOCI(url string) bool {
 	return strings.HasPrefix(url, fmt.Sprintf("%s://", OCIScheme))
 }
 
+// ContainsTag determines whether a tag is found in a provided list of tags
+func ContainsTag(tags []string, tag string) bool {
+	for _, t := range tags {
+		if tag == t {
+			return true
+		}
+	}
+	return false
+}
+
+func GetTagMatchingVersionOrConstraint(tags []string, versionString string) (string, error) {
+	var constraint *semver.Constraints
+	if versionString == "" {
+		// If string is empty, set wildcard constraint
+		constraint, _ = semver.NewConstraint("*")
+	} else {
+		// when customer input exact version, check whether have exact match
+		// one first
+		for _, v := range tags {
+			if versionString == v {
+				return v, nil
+			}
+		}
+
+		// Otherwise set constraint to the string given
+		var err error
+		constraint, err = semver.NewConstraint(versionString)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	// Otherwise try to find the first available version matching the string,
+	// in case it is a constraint
+	for _, v := range tags {
+		test, err := semver.NewVersion(v)
+		if err != nil {
+			continue
+		}
+		if constraint.Check(test) {
+			return v, nil
+		}
+	}
+
+	return "", errors.Errorf("Could not locate a version matching provided version string %s", versionString)
+}
+
 // extractChartMeta is used to extract a chart metadata from a byte array
 func extractChartMeta(chartData []byte) (*chart.Metadata, error) {
 	ch, err := loader.LoadArchive(bytes.NewReader(chartData))

internal/resolver/resolver.go

@@ -18,6 +18,7 @@ package resolver
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -39,15 +40,17 @@ const FeatureGateOCI = gates.Gate("HELM_EXPERIMENTAL_OCI")
 
 // Resolver resolves dependencies from semantic version ranges to a particular version.
 type Resolver struct {
-	chartpath string
+	chartpath      string
-	cachepath string
+	cachepath      string
+	registryClient *registry.Client
 }
 
-// New creates a new resolver for a given chart and a given helm home.
+// New creates a new resolver for a given chart, helm home and registry client.
-func New(chartpath, cachepath string) *Resolver {
+func New(chartpath, cachepath string, registryClient *registry.Client) *Resolver {
 	return &Resolver{
-		chartpath: chartpath,
+		chartpath:      chartpath,
-		cachepath: cachepath,
+		cachepath:      cachepath,
+		registryClient: registryClient,
 	}
 }
 
@@ -139,6 +142,24 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string
 				return nil, errors.Wrapf(FeatureGateOCI.Error(),
 					"repository %s is an OCI registry", d.Repository)
 			}
+
+			// Retrieve list of tags for repository
+			ref := fmt.Sprintf("%s/%s", strings.TrimPrefix(d.Repository, fmt.Sprintf("%s://", registry.OCIScheme)), d.Name)
+			tags, err := r.registryClient.Tags(ref)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not retrieve list of tags for repository %s", d.Repository)
+			}
+
+			vs = make(repo.ChartVersions, len(tags))
+			for ti, t := range tags {
+				// Mock chart version objects
+				version := &repo.ChartVersion{
+					Metadata: &chart.Metadata{
+						Version: t,
+					},
+				}
+				vs[ti] = version
+			}
 		}
 
 		locked[i] = &chart.Dependency{
@@ -149,7 +170,8 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string
 		// The version are already sorted and hence the first one to satisfy the constraint is used
 		for _, ver := range vs {
 			v, err := semver.NewVersion(ver.Version)
-			if err != nil || len(ver.URLs) == 0 {
+			// OCI does not need URLs
+			if err != nil || (!registry.IsOCI(d.Repository) && len(ver.URLs) == 0) {
 				// Not a legit entry.
 				continue
 			}

internal/resolver/resolver_test.go

@@ -19,6 +19,7 @@ import (
 	"runtime"
 	"testing"
 
+	"helm.sh/helm/v3/internal/experimental/registry"
 	"helm.sh/helm/v3/pkg/chart"
 )
 
@@ -139,7 +140,8 @@ func TestResolve(t *testing.T) {
 	}
 
 	repoNames := map[string]string{"alpine": "kubernetes-charts", "redis": "kubernetes-charts"}
-	r := New("testdata/chartpath", "testdata/repository")
+	registryClient, _ := registry.NewClient()
+	r := New("testdata/chartpath", "testdata/repository", registryClient)
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			l, err := r.Resolve(tt.req, repoNames)

pkg/action/install.go

@@ -695,10 +695,9 @@ func (c *ChartPathOptions) LocateChart(name string, settings *cli.EnvSettings) (
 	}
 
 	if registry.IsOCI(name) {
-		if version == "" {
+		if version != "" {
-			return "", errors.New("version is explicitly required for OCI registries")
+			dl.Options = append(dl.Options, getter.WithTagName(version))
 		}
-		dl.Options = append(dl.Options, getter.WithTagName(version))
 	}
 
 	if c.Verify {

pkg/action/pull.go

@@ -87,18 +87,14 @@ func (p *Pull) Run(chartRef string) (string, error) {
 			getter.WithTLSClientConfig(p.CertFile, p.KeyFile, p.CaFile),
 			getter.WithInsecureSkipVerifyTLS(p.InsecureSkipTLSverify),
 		},
+		RegistryClient:   p.cfg.RegistryClient,
 		RepositoryConfig: p.Settings.RepositoryConfig,
 		RepositoryCache:  p.Settings.RepositoryCache,
 	}
 
 	if registry.IsOCI(chartRef) {
-		if p.Version == "" {
-			return out.String(), errors.Errorf("--version flag is explicitly required for OCI registries")
-		}
-
 		c.Options = append(c.Options,
-			getter.WithRegistryClient(p.cfg.RegistryClient),
+			getter.WithRegistryClient(p.cfg.RegistryClient))
-			getter.WithTagName(p.Version))
 	}
 
 	if p.Verify {

pkg/downloader/chart_downloader.go

@@ -103,7 +103,8 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven
 
 	name := filepath.Base(u.Path)
 	if u.Scheme == registry.OCIScheme {
-		name = fmt.Sprintf("%s-%s.tgz", name, version)
+		idx := strings.LastIndexByte(name, ':')
+		name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:])
 	}
 
 	destfile := filepath.Join(dest, name)
@@ -139,12 +140,37 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven
 	return destfile, ver, nil
 }
 
+func (c *ChartDownloader) getOciURI(ref, version string, u *url.URL) (*url.URL, error) {
+	// Retrieve list of repository tags
+	tags, err := c.RegistryClient.Tags(strings.TrimPrefix(ref, fmt.Sprintf("%s://", registry.OCIScheme)))
+	if err != nil {
+		return nil, err
+	}
+	if len(tags) == 0 {
+		return nil, errors.Errorf("Unable to locate any tags in provided repository: %s", ref)
+	}
+
+	// Determine if version provided
+	// If empty, try to get the highest available tag
+	// If exact version, try to find it
+	// If semver constraint string, try to find a match
+	tag, err := registry.GetTagMatchingVersionOrConstraint(tags, version)
+	if err != nil {
+		return nil, err
+	}
+
+	u.Path = fmt.Sprintf("%s:%s", u.Path, tag)
+
+	return u, err
+}
+
 // ResolveChartVersion resolves a chart reference to a URL.
 //
 // It returns the URL and sets the ChartDownloader's Options that can fetch
 // the URL using the appropriate Getter.
 //
-// A reference may be an HTTP URL, a 'reponame/chartname' reference, or a local path.
+// A reference may be an HTTP URL, an oci reference URL, a 'reponame/chartname'
+// reference, or a local path.
 //
 // A version is a SemVer string (1.2.3-beta.1+f334a6789).
 //
@@ -159,6 +185,10 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, er
 		return nil, errors.Errorf("invalid chart URL format: %s", ref)
 	}
 
+	if registry.IsOCI(u.String()) {
+		return c.getOciURI(ref, version, u)
+	}
+
 	rf, err := loadRepoConfig(c.RepositoryConfig)
 	if err != nil {
 		return u, err

pkg/downloader/manager.go

@@ -232,7 +232,7 @@ func (m *Manager) loadChartDir() (*chart.Chart, error) {
 //
 // This returns a lock file, which has all of the dependencies normalized to a specific version.
 func (m *Manager) resolve(req []*chart.Dependency, repoNames map[string]string) (*chart.Lock, error) {
-	res := resolver.New(m.ChartPath, m.RepositoryCache)
+	res := resolver.New(m.ChartPath, m.RepositoryCache, m.RegistryClient)
 	return res.Resolve(req, repoNames)
 }
 
@@ -332,6 +332,7 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error {
 			Keyring:          m.Keyring,
 			RepositoryConfig: m.RepositoryConfig,
 			RepositoryCache:  m.RepositoryCache,
+			RegistryClient:   m.RegistryClient,
 			Getters:          m.Getters,
 			Options: []getter.Option{
 				getter.WithBasicAuth(username, password),

pkg/getter/ocigetter.go

@@ -28,7 +28,7 @@ type OCIGetter struct {
 	opts options
 }
 
-//Get performs a Get from repo.Getter and returns the body.
+// Get performs a Get from repo.Getter and returns the body.
 func (g *OCIGetter) Get(href string, options ...Option) (*bytes.Buffer, error) {
 	for _, opt := range options {
 		opt(&g.opts)
@@ -50,10 +50,6 @@ func (g *OCIGetter) get(href string) (*bytes.Buffer, error) {
 			registry.PullOptWithProv(true))
 	}
 
-	if version := g.opts.version; version != "" {
-		ref = fmt.Sprintf("%s:%s", ref, version)
-	}
-
 	result, err := client.Pull(ref, pullOpts...)
 	if err != nil {
 		return nil, err