Merge pull request #8843 from bloodorangeio/oci-pull

Signed-off-by: Matt Farina <matt@mattfarina.com>
pull/9192/head
Matt Farina 4 years ago committed by GitHub
commit 49f895db6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -82,7 +82,7 @@ the contents of a chart.
This will produce an error if the chart cannot be loaded.
`
func newDependencyCmd(out io.Writer) *cobra.Command {
func newDependencyCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
cmd := &cobra.Command{
Use: "dependency update|build|list",
Aliases: []string{"dep", "dependencies"},
@ -92,7 +92,7 @@ func newDependencyCmd(out io.Writer) *cobra.Command {
}
cmd.AddCommand(newDependencyListCmd(out))
cmd.AddCommand(newDependencyUpdateCmd(out))
cmd.AddCommand(newDependencyUpdateCmd(cfg, out))
cmd.AddCommand(newDependencyBuildCmd(out))
return cmd

@ -43,7 +43,7 @@ in the Chart.yaml file, but (b) at the wrong version.
`
// newDependencyUpdateCmd creates a new dependency update command.
func newDependencyUpdateCmd(out io.Writer) *cobra.Command {
func newDependencyUpdateCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
client := action.NewDependency()
cmd := &cobra.Command{
@ -63,6 +63,7 @@ func newDependencyUpdateCmd(out io.Writer) *cobra.Command {
Keyring: client.Keyring,
SkipUpdate: client.SkipRefresh,
Getters: getter.All(settings),
RegistryClient: cfg.RegistryClient,
RepositoryConfig: settings.RepositoryConfig,
RepositoryCache: settings.RepositoryCache,
Debug: settings.Debug,

@ -40,6 +40,23 @@ func TestDependencyUpdateCmd(t *testing.T) {
defer srv.Stop()
t.Logf("Listening on directory %s", srv.Root())
ociSrv, err := repotest.NewOCIServer(t, srv.Root())
if err != nil {
t.Fatal(err)
}
ociChartName := "oci-depending-chart"
c := createTestingMetadataForOCI(ociChartName, ociSrv.RegistryURL)
if err := chartutil.SaveDir(c, ociSrv.Dir); err != nil {
t.Fatal(err)
}
ociSrv.Run(t, repotest.WithDependingChart(c))
err = os.Setenv("HELM_EXPERIMENTAL_OCI", "1")
if err != nil {
t.Fatal("failed to set environment variable enabling OCI support")
}
if err := srv.LinkIndices(); err != nil {
t.Fatal(err)
}
@ -115,6 +132,22 @@ func TestDependencyUpdateCmd(t *testing.T) {
if _, err := os.Stat(unexpected); err == nil {
t.Fatalf("Unexpected %q", unexpected)
}
// test for OCI charts
cmd := fmt.Sprintf("dependency update '%s' --repository-config %s --repository-cache %s --registry-config %s/config.json",
dir(ociChartName),
dir("repositories.yaml"),
dir(),
dir())
_, out, err = executeActionCommand(cmd)
if err != nil {
t.Logf("Output: %s", out)
t.Fatal(err)
}
expect = dir(ociChartName, "charts/oci-dependent-chart-0.1.0.tgz")
if _, err := os.Stat(expect); err != nil {
t.Fatal(err)
}
}
func TestDependencyUpdateCmd_DontDeleteOldChartsOnError(t *testing.T) {
@ -193,6 +226,19 @@ func createTestingMetadata(name, baseURL string) *chart.Chart {
}
}
func createTestingMetadataForOCI(name, registryURL string) *chart.Chart {
return &chart.Chart{
Metadata: &chart.Metadata{
APIVersion: chart.APIVersionV2,
Name: name,
Version: "1.2.3",
Dependencies: []*chart.Dependency{
{Name: "oci-dependent-chart", Version: "0.1.0", Repository: fmt.Sprintf("oci://%s/u/ocitestuser", registryURL)},
},
},
}
}
// createTestingChart creates a basic chart that depends on reqtest-0.1.0
//
// The baseURL can be used to point to a particular repository server.

@ -20,6 +20,7 @@ import (
"fmt"
"io"
"log"
"strings"
"github.com/spf13/cobra"
@ -42,8 +43,8 @@ file, and MUST pass the verification process. Failure in any part of this will
result in an error, and the chart will not be saved locally.
`
func newPullCmd(out io.Writer) *cobra.Command {
client := action.NewPull()
func newPullCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
client := action.NewPullWithOpts(action.WithConfig(cfg))
cmd := &cobra.Command{
Use: "pull [chart URL | repo/chartname] [...]",
@ -64,6 +65,12 @@ func newPullCmd(out io.Writer) *cobra.Command {
client.Version = ">0.0.0-0"
}
if strings.HasPrefix(args[0], "oci://") {
if !FeatureGateOCI.IsEnabled() {
return FeatureGateOCI.Error()
}
}
for i := 0; i < len(args); i++ {
output, err := client.Run(args[i])
if err != nil {

@ -32,6 +32,13 @@ func TestPullCmd(t *testing.T) {
}
defer srv.Stop()
os.Setenv("HELM_EXPERIMENTAL_OCI", "1")
ociSrv, err := repotest.NewOCIServer(t, srv.Root())
if err != nil {
t.Fatal(err)
}
ociSrv.Run(t)
if err := srv.LinkIndices(); err != nil {
t.Fatal(err)
}
@ -139,23 +146,70 @@ func TestPullCmd(t *testing.T) {
failExpect: "Failed to fetch chart version",
wantError: true,
},
{
name: "Fetch OCI Chart",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart --version 0.1.0", ociSrv.RegistryURL),
expectFile: "./oci-dependent-chart-0.1.0.tgz",
},
{
name: "Fetch OCI Chart with untar",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart --version 0.1.0 --untar", ociSrv.RegistryURL),
expectFile: "./oci-dependent-chart",
expectDir: true,
},
{
name: "Fetch OCI Chart with untar and untardir",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart --version 0.1.0 --untar --untardir ocitest2", ociSrv.RegistryURL),
expectFile: "./ocitest2",
expectDir: true,
},
{
name: "OCI Fetch untar when dir with same name existed",
args: fmt.Sprintf("oci-test-chart oci://%s/u/ocitestuser/oci-dependent-chart --version 0.1.0 --untar --untardir ocitest2 --untar --untardir ocitest2", ociSrv.RegistryURL),
wantError: true,
wantErrorMsg: fmt.Sprintf("failed to untar: a file or directory with the name %s already exists", filepath.Join(srv.Root(), "ocitest2")),
},
{
name: "Fail fetching non-existent OCI chart",
args: fmt.Sprintf("oci://%s/u/ocitestuser/nosuchthing --version 0.1.0", ociSrv.RegistryURL),
failExpect: "Failed to fetch",
wantError: true,
},
{
name: "Fail fetching OCI chart without version specified",
args: fmt.Sprintf("oci://%s/u/ocitestuser/nosuchthing", ociSrv.RegistryURL),
wantErrorMsg: "Error: --version flag is explicitly required for OCI registries",
wantError: true,
},
{
name: "Fail fetching OCI chart without version specified",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart:0.1.0", ociSrv.RegistryURL),
wantErrorMsg: "Error: --version flag is explicitly required for OCI registries",
wantError: true,
},
{
name: "Fail fetching OCI chart without version specified",
args: fmt.Sprintf("oci://%s/u/ocitestuser/oci-dependent-chart:0.1.0 --version 0.1.0", ociSrv.RegistryURL),
wantError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
outdir := srv.Root()
cmd := fmt.Sprintf("fetch %s -d '%s' --repository-config %s --repository-cache %s ",
cmd := fmt.Sprintf("fetch %s -d '%s' --repository-config %s --repository-cache %s --registry-config %s",
tt.args,
outdir,
filepath.Join(outdir, "repositories.yaml"),
outdir,
filepath.Join(outdir, "config.json"),
)
// Create file or Dir before helm pull --untar, see: https://github.com/helm/helm/issues/7182
if tt.existFile != "" {
file := filepath.Join(outdir, tt.existFile)
_, err := os.Create(file)
if err != nil {
t.Fatal("err")
t.Fatal(err)
}
}
if tt.existDir != "" {

@ -153,12 +153,22 @@ func newRootCmd(actionConfig *action.Configuration, out io.Writer, args []string
flags.ParseErrorsWhitelist.UnknownFlags = true
flags.Parse(args)
registryClient, err := registry.NewClient(
registry.ClientOptDebug(settings.Debug),
registry.ClientOptWriter(out),
registry.ClientOptCredentialsFile(settings.RegistryConfig),
)
if err != nil {
return nil, err
}
actionConfig.RegistryClient = registryClient
// Add subcommands
cmd.AddCommand(
// chart commands
newCreateCmd(out),
newDependencyCmd(out),
newPullCmd(out),
newDependencyCmd(actionConfig, out),
newPullCmd(actionConfig, out),
newShowCmd(out),
newLintCmd(out),
newPackageCmd(out),
@ -188,15 +198,6 @@ func newRootCmd(actionConfig *action.Configuration, out io.Writer, args []string
)
// Add *experimental* subcommands
registryClient, err := registry.NewClient(
registry.ClientOptDebug(settings.Debug),
registry.ClientOptWriter(out),
registry.ClientOptCredentialsFile(settings.RegistryConfig),
)
if err != nil {
return nil, err
}
actionConfig.RegistryClient = registryClient
cmd.AddCommand(
newRegistryCmd(actionConfig, out),
newChartCmd(actionConfig, out),

@ -27,6 +27,7 @@ require (
github.com/mitchellh/copystructure v1.0.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.1
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
github.com/pkg/errors v0.9.1
github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351
github.com/sirupsen/logrus v1.7.0

@ -17,6 +17,7 @@ limitations under the License.
package registry // import "helm.sh/helm/v3/internal/experimental/registry"
import (
"bytes"
"context"
"fmt"
"io"
@ -25,6 +26,7 @@ import (
"sort"
auth "github.com/deislabs/oras/pkg/auth/docker"
"github.com/deislabs/oras/pkg/content"
"github.com/deislabs/oras/pkg/oras"
"github.com/gosuri/uitable"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -144,7 +146,60 @@ func (c *Client) PushChart(ref *Reference) error {
}
// PullChart downloads a chart from a registry
func (c *Client) PullChart(ref *Reference) error {
func (c *Client) PullChart(ref *Reference) (*bytes.Buffer, error) {
buf := bytes.NewBuffer(nil)
if ref.Tag == "" {
return buf, errors.New("tag explicitly required")
}
fmt.Fprintf(c.out, "%s: Pulling from %s\n", ref.Tag, ref.Repo)
store := content.NewMemoryStore()
fullname := ref.FullName()
_ = fullname
_, layerDescriptors, err := oras.Pull(ctx(c.out, c.debug), c.resolver, ref.FullName(), store,
oras.WithPullEmptyNameAllowed(),
oras.WithAllowedMediaTypes(KnownMediaTypes()))
if err != nil {
return buf, err
}
numLayers := len(layerDescriptors)
if numLayers < 1 {
return buf, errors.New(
fmt.Sprintf("manifest does not contain at least 1 layer (total: %d)", numLayers))
}
var contentLayer *ocispec.Descriptor
for _, layer := range layerDescriptors {
layer := layer
switch layer.MediaType {
case HelmChartContentLayerMediaType:
contentLayer = &layer
}
}
if contentLayer == nil {
return buf, errors.New(
fmt.Sprintf("manifest does not contain a layer with mediatype %s",
HelmChartContentLayerMediaType))
}
_, b, ok := store.Get(*contentLayer)
if !ok {
return buf, errors.Errorf("Unable to retrieve blob with digest %s", contentLayer.Digest)
}
buf = bytes.NewBuffer(b)
return buf, nil
}
// PullChartToCache pulls a chart from an OCI Registry to the Registry Cache.
// This function is needed for `helm chart pull`, which is experimental and will be deprecated soon.
// Likewise, the Registry cache will soon be deprecated as will this function.
func (c *Client) PullChartToCache(ref *Reference) error {
if ref.Tag == "" {
return errors.New("tag explicitly required")
}

@ -22,7 +22,6 @@ import (
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"net/url"
@ -33,12 +32,12 @@ import (
"time"
"github.com/containerd/containerd/errdefs"
auth "github.com/deislabs/oras/pkg/auth/docker"
"github.com/docker/distribution/configuration"
"github.com/docker/distribution/registry"
_ "github.com/docker/distribution/registry/auth/htpasswd"
_ "github.com/docker/distribution/registry/storage/driver/inmemory"
"github.com/phayes/freeport"
"github.com/stretchr/testify/suite"
"golang.org/x/crypto/bcrypt"
@ -107,7 +106,7 @@ func (suite *RegistryClientTestSuite) SetupSuite() {
// Registry config
config := &configuration.Configuration{}
port, err := getFreePort()
port, err := freeport.GetFreePort()
suite.Nil(err, "no error finding free port for test registry")
suite.DockerRegistryHost = fmt.Sprintf("localhost:%d", port)
config.HTTP.Addr = fmt.Sprintf(":%d", port)
@ -202,13 +201,13 @@ func (suite *RegistryClientTestSuite) Test_4_PullChart() {
// non-existent ref
ref, err := ParseReference(fmt.Sprintf("%s/testrepo/whodis:9.9.9", suite.DockerRegistryHost))
suite.Nil(err)
err = suite.RegistryClient.PullChart(ref)
_, err = suite.RegistryClient.PullChart(ref)
suite.NotNil(err)
// existing ref
ref, err = ParseReference(fmt.Sprintf("%s/testrepo/testchart:1.2.3", suite.DockerRegistryHost))
suite.Nil(err)
err = suite.RegistryClient.PullChart(ref)
_, err = suite.RegistryClient.PullChart(ref)
suite.Nil(err)
}
@ -245,7 +244,7 @@ func (suite *RegistryClientTestSuite) Test_8_ManInTheMiddle() {
suite.Nil(err)
// returns content that does not match the expected digest
err = suite.RegistryClient.PullChart(ref)
_, err = suite.RegistryClient.PullChart(ref)
suite.NotNil(err)
suite.True(errdefs.IsFailedPrecondition(err))
}
@ -254,21 +253,6 @@ func TestRegistryClientTestSuite(t *testing.T) {
suite.Run(t, new(RegistryClientTestSuite))
}
// borrowed from https://github.com/phayes/freeport
func getFreePort() (int, error) {
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
if err != nil {
return 0, err
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return 0, err
}
defer l.Close()
return l.Addr().(*net.TCPAddr).Port, nil
}
func initCompromisedRegistryTestServer() string {
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.Contains(r.URL.Path, "manifests") {

@ -28,11 +28,14 @@ import (
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/gates"
"helm.sh/helm/v3/pkg/helmpath"
"helm.sh/helm/v3/pkg/provenance"
"helm.sh/helm/v3/pkg/repo"
)
const FeatureGateOCI = gates.Gate("HELM_EXPERIMENTAL_OCI")
// Resolver resolves dependencies from semantic version ranges to a particular version.
type Resolver struct {
chartpath string
@ -88,6 +91,7 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string
}
continue
}
constraint, err := semver.NewConstraint(d.Version)
if err != nil {
return nil, errors.Wrapf(err, "dependency %q has an invalid version/constraint format", d.Name)
@ -104,21 +108,34 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string
continue
}
repoIndex, err := repo.LoadIndexFile(filepath.Join(r.cachepath, helmpath.CacheIndexFile(repoName)))
if err != nil {
return nil, errors.Wrapf(err, "no cached repository for %s found. (try 'helm repo update')", repoName)
}
var vs repo.ChartVersions
var version string
var ok bool
found := true
if !strings.HasPrefix(d.Repository, "oci://") {
repoIndex, err := repo.LoadIndexFile(filepath.Join(r.cachepath, helmpath.CacheIndexFile(repoName)))
if err != nil {
return nil, errors.Wrapf(err, "no cached repository for %s found. (try 'helm repo update')", repoName)
}
vs, ok := repoIndex.Entries[d.Name]
if !ok {
return nil, errors.Errorf("%s chart not found in repo %s", d.Name, d.Repository)
vs, ok = repoIndex.Entries[d.Name]
if !ok {
return nil, errors.Errorf("%s chart not found in repo %s", d.Name, d.Repository)
}
found = false
} else {
version = d.Version
if !FeatureGateOCI.IsEnabled() {
return nil, errors.Wrapf(FeatureGateOCI.Error(),
"repository %s is an OCI registry", d.Repository)
}
}
locked[i] = &chart.Dependency{
Name: d.Name,
Repository: d.Repository,
Version: version,
}
found := false
// The version are already sorted and hence the first one to satisfy the constraint is used
for _, ver := range vs {
v, err := semver.NewVersion(ver.Version)

@ -40,5 +40,5 @@ func (a *ChartPull) Run(out io.Writer, ref string) error {
if err != nil {
return err
}
return a.cfg.RegistryClient.PullChart(r)
return a.cfg.RegistryClient.PullChartToCache(r)
}

@ -45,11 +45,30 @@ type Pull struct {
VerifyLater bool
UntarDir string
DestDir string
cfg *Configuration
}
// NewPull creates a new Pull object with the given configuration.
type PullOpt func(*Pull)
func WithConfig(cfg *Configuration) PullOpt {
return func(p *Pull) {
p.cfg = cfg
}
}
// NewPull creates a new Pull object.
func NewPull() *Pull {
return &Pull{}
return NewPullWithOpts()
}
// NewPull creates a new pull, with configuration options.
func NewPullWithOpts(opts ...PullOpt) *Pull {
p := &Pull{}
for _, fn := range opts {
fn(p)
}
return p
}
// Run executes 'helm pull' against the given release.
@ -70,6 +89,16 @@ func (p *Pull) Run(chartRef string) (string, error) {
RepositoryCache: p.Settings.RepositoryCache,
}
if strings.HasPrefix(chartRef, "oci://") {
if p.Version == "" {
return out.String(), errors.Errorf("--version flag is explicitly required for OCI registries")
}
c.Options = append(c.Options,
getter.WithRegistryClient(p.cfg.RegistryClient),
getter.WithTagName(p.Version))
}
if p.Verify {
c.Verify = downloader.VerifyAlways
} else if p.VerifyLater {
@ -123,6 +152,7 @@ func (p *Pull) Run(chartRef string) (string, error) {
_, chartName := filepath.Split(chartRef)
udCheck = filepath.Join(udCheck, chartName)
}
if _, err := os.Stat(udCheck); err != nil {
if err := os.MkdirAll(udCheck, 0755); err != nil {
return out.String(), errors.Wrap(err, "failed to untar (mkdir)")

@ -25,6 +25,7 @@ import (
"github.com/pkg/errors"
"helm.sh/helm/v3/internal/experimental/registry"
"helm.sh/helm/v3/internal/fileutil"
"helm.sh/helm/v3/internal/urlutil"
"helm.sh/helm/v3/pkg/getter"
@ -68,6 +69,7 @@ type ChartDownloader struct {
Getters getter.Providers
// Options provide parameters to be passed along to the Getter being initialized.
Options []getter.Option
RegistryClient *registry.Client
RepositoryConfig string
RepositoryCache string
}
@ -100,6 +102,10 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven
}
name := filepath.Base(u.Path)
if u.Scheme == "oci" {
name = fmt.Sprintf("%s-%s.tgz", name, version)
}
destfile := filepath.Join(dest, name)
if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil {
return destfile, nil, err

@ -26,6 +26,7 @@ import (
"os"
"path"
"path/filepath"
"regexp"
"strings"
"sync"
@ -33,6 +34,7 @@ import (
"github.com/pkg/errors"
"sigs.k8s.io/yaml"
"helm.sh/helm/v3/internal/experimental/registry"
"helm.sh/helm/v3/internal/resolver"
"helm.sh/helm/v3/internal/third_party/dep/fs"
"helm.sh/helm/v3/internal/urlutil"
@ -71,6 +73,7 @@ type Manager struct {
SkipUpdate bool
// Getter collection for the operation
Getters []getter.Provider
RegistryClient *registry.Client
RepositoryConfig string
RepositoryCache string
}
@ -332,7 +335,24 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error {
},
}
if _, _, err := dl.DownloadTo(churl, "", destPath); err != nil {
version := ""
if strings.HasPrefix(churl, "oci://") {
if !resolver.FeatureGateOCI.IsEnabled() {
return errors.Wrapf(resolver.FeatureGateOCI.Error(),
"the repository %s is an OCI registry", churl)
}
churl, version, err = parseOCIRef(churl)
if err != nil {
return errors.Wrapf(err, "could not parse OCI reference")
}
dl.Options = append(dl.Options,
getter.WithRegistryClient(m.RegistryClient),
getter.WithTagName(version))
}
_, _, err = dl.DownloadTo(churl, version, destPath)
if err != nil {
saveError = errors.Wrapf(err, "could not download %s", churl)
break
}
@ -375,6 +395,18 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error {
return nil
}
func parseOCIRef(chartRef string) (string, string, error) {
refTagRegexp := regexp.MustCompile(`^(oci://[^:]+(:[0-9]{1,5})?[^:]+):(.*)$`)
caps := refTagRegexp.FindStringSubmatch(chartRef)
if len(caps) != 4 {
return "", "", errors.Errorf("improperly formatted oci chart reference: %s", chartRef)
}
chartRef = caps[1]
tag := caps[3]
return chartRef, tag, nil
}
// safeDeleteDep deletes any versions of the given dependency in the given directory.
//
// It does this by first matching the file name to an expected pattern, then loading
@ -539,6 +571,11 @@ func (m *Manager) resolveRepoNames(deps []*chart.Dependency) (map[string]string,
continue
}
if strings.HasPrefix(dd.Repository, "oci://") {
reposMap[dd.Name] = dd.Repository
continue
}
found := false
for _, repo := range repos {
@ -648,7 +685,12 @@ func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error {
//
// If it finds a URL that is "relative", it will prepend the repoURL.
func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (url, username, password string, err error) {
if strings.HasPrefix(repoURL, "oci://") {
return fmt.Sprintf("%s/%s:%s", repoURL, name, version), "", "", nil
}
for _, cr := range repos {
if urlutil.Equal(repoURL, cr.Config.URL) {
var entry repo.ChartVersions
entry, err = findEntryByName(name, cr)
@ -671,10 +713,10 @@ func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*
}
url, err = repo.FindChartInRepoURL(repoURL, name, version, "", "", "", m.Getters)
if err == nil {
return
return url, username, password, err
}
err = errors.Errorf("chart %s not found in %s: %s", name, repoURL, err)
return
return url, username, password, err
}
// findEntryByName finds an entry in the chart repository whose name matches the given name.

@ -22,6 +22,7 @@ import (
"github.com/pkg/errors"
"helm.sh/helm/v3/internal/experimental/registry"
"helm.sh/helm/v3/pkg/cli"
)
@ -33,10 +34,13 @@ type options struct {
certFile string
keyFile string
caFile string
unTar bool
insecureSkipVerifyTLS bool
username string
password string
userAgent string
version string
registryClient *registry.Client
timeout time.Duration
}
@ -90,6 +94,24 @@ func WithTimeout(timeout time.Duration) Option {
}
}
func WithTagName(tagname string) Option {
return func(opts *options) {
opts.version = tagname
}
}
func WithRegistryClient(client *registry.Client) Option {
return func(opts *options) {
opts.registryClient = client
}
}
func WithUntar() Option {
return func(opts *options) {
opts.unTar = true
}
}
// Getter is an interface to support GET to the specified URL.
type Getter interface {
// Get file content by url string
@ -139,11 +161,16 @@ var httpProvider = Provider{
New: NewHTTPGetter,
}
var ociProvider = Provider{
Schemes: []string{"oci"},
New: NewOCIGetter,
}
// All finds all of the registered getters as a list of Provider instances.
// Currently, the built-in getters and the discovered plugins with downloader
// notations are collected.
func All(settings *cli.EnvSettings) Providers {
result := Providers{httpProvider}
result := Providers{httpProvider, ociProvider}
pluginDownloaders, _ := collectPlugins(settings)
result = append(result, pluginDownloaders...)
return result

@ -57,8 +57,8 @@ func TestAll(t *testing.T) {
env.PluginsDirectory = pluginDir
all := All(env)
if len(all) != 3 {
t.Errorf("expected 3 providers (default plus two plugins), got %d", len(all))
if len(all) != 4 {
t.Errorf("expected 4 providers (default plus three plugins), got %d", len(all))
}
if _, err := all.ByScheme("test2"); err != nil {

@ -0,0 +1,69 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package getter
import (
"bytes"
"fmt"
"strings"
"helm.sh/helm/v3/internal/experimental/registry"
)
// OCIGetter is the default HTTP(/S) backend handler
type OCIGetter struct {
opts options
}
//Get performs a Get from repo.Getter and returns the body.
func (g *OCIGetter) Get(href string, options ...Option) (*bytes.Buffer, error) {
for _, opt := range options {
opt(&g.opts)
}
return g.get(href)
}
func (g *OCIGetter) get(href string) (*bytes.Buffer, error) {
client := g.opts.registryClient
ref := strings.TrimPrefix(href, "oci://")
if version := g.opts.version; version != "" {
ref = fmt.Sprintf("%s:%s", ref, version)
}
r, err := registry.ParseReference(ref)
if err != nil {
return nil, err
}
buf, err := client.PullChart(r)
if err != nil {
return nil, err
}
return buf, nil
}
// NewOCIGetter constructs a valid http/https client as a Getter
func NewOCIGetter(options ...Option) (Getter, error) {
var client OCIGetter
for _, opt := range options {
opt(&client.opts)
}
return &client, nil
}

@ -16,17 +16,30 @@ limitations under the License.
package repotest
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
"helm.sh/helm/v3/internal/tlsutil"
auth "github.com/deislabs/oras/pkg/auth/docker"
"github.com/docker/distribution/configuration"
"github.com/docker/distribution/registry"
_ "github.com/docker/distribution/registry/auth/htpasswd" // used for docker test registry
_ "github.com/docker/distribution/registry/storage/driver/inmemory" // used for docker test registry
"github.com/phayes/freeport"
"golang.org/x/crypto/bcrypt"
"sigs.k8s.io/yaml"
ociRegistry "helm.sh/helm/v3/internal/experimental/registry"
"helm.sh/helm/v3/internal/tlsutil"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
"helm.sh/helm/v3/pkg/repo"
)
@ -43,6 +56,166 @@ func NewTempServerWithCleanup(t *testing.T, glob string) (*Server, error) {
return srv, err
}
type OCIServer struct {
*registry.Registry
RegistryURL string
Dir string
TestUsername string
TestPassword string
Client *ociRegistry.Client
}
type OCIServerRunConfig struct {
DependingChart *chart.Chart
}
type OCIServerOpt func(config *OCIServerRunConfig)
func WithDependingChart(c *chart.Chart) OCIServerOpt {
return func(config *OCIServerRunConfig) {
config.DependingChart = c
}
}
func NewOCIServer(t *testing.T, dir string) (*OCIServer, error) {
testHtpasswdFileBasename := "authtest.htpasswd"
testUsername, testPassword := "username", "password"
pwBytes, err := bcrypt.GenerateFromPassword([]byte(testPassword), bcrypt.DefaultCost)
if err != nil {
t.Fatal("error generating bcrypt password for test htpasswd file")
}
htpasswdPath := filepath.Join(dir, testHtpasswdFileBasename)
err = ioutil.WriteFile(htpasswdPath, []byte(fmt.Sprintf("%s:%s\n", testUsername, string(pwBytes))), 0644)
if err != nil {
t.Fatalf("error creating test htpasswd file")
}
// Registry config
config := &configuration.Configuration{}
port, err := freeport.GetFreePort()
if err != nil {
t.Fatalf("error finding free port for test registry")
}
config.HTTP.Addr = fmt.Sprintf(":%d", port)
config.HTTP.DrainTimeout = time.Duration(10) * time.Second
config.Storage = map[string]configuration.Parameters{"inmemory": map[string]interface{}{}}
config.Auth = configuration.Auth{
"htpasswd": configuration.Parameters{
"realm": "localhost",
"path": htpasswdPath,
},
}
registryURL := fmt.Sprintf("localhost:%d", port)
r, err := registry.NewRegistry(context.Background(), config)
if err != nil {
t.Fatal(err)
}
return &OCIServer{
Registry: r,
RegistryURL: registryURL,
TestUsername: testUsername,
TestPassword: testPassword,
Dir: dir,
}, nil
}
func (srv *OCIServer) Run(t *testing.T, opts ...OCIServerOpt) {
cfg := &OCIServerRunConfig{}
for _, fn := range opts {
fn(cfg)
}
go srv.ListenAndServe()
credentialsFile := filepath.Join(srv.Dir, "config.json")
client, err := auth.NewClient(credentialsFile)
if err != nil {
t.Fatalf("error creating auth client")
}
resolver, err := client.Resolver(context.Background(), http.DefaultClient, false)
if err != nil {
t.Fatalf("error creating resolver")
}
// init test client
registryClient, err := ociRegistry.NewClient(
ociRegistry.ClientOptDebug(true),
ociRegistry.ClientOptWriter(os.Stdout),
ociRegistry.ClientOptAuthorizer(&ociRegistry.Authorizer{
Client: client,
}),
ociRegistry.ClientOptResolver(&ociRegistry.Resolver{
Resolver: resolver,
}),
)
if err != nil {
t.Fatalf("error creating registry client")
}
err = registryClient.Login(srv.RegistryURL, srv.TestUsername, srv.TestPassword, false)
if err != nil {
t.Fatalf("error logging into registry with good credentials")
}
ref, err := ociRegistry.ParseReference(fmt.Sprintf("%s/u/ocitestuser/oci-dependent-chart:0.1.0", srv.RegistryURL))
if err != nil {
t.Fatalf("")
}
err = chartutil.ExpandFile(srv.Dir, filepath.Join(srv.Dir, "oci-dependent-chart-0.1.0.tgz"))
if err != nil {
t.Fatal(err)
}
// valid chart
ch, err := loader.LoadDir(filepath.Join(srv.Dir, "oci-dependent-chart"))
if err != nil {
t.Fatal("error loading chart")
}
err = os.RemoveAll(filepath.Join(srv.Dir, "oci-dependent-chart"))
if err != nil {
t.Fatal("error removing chart before push")
}
err = registryClient.SaveChart(ch, ref)
if err != nil {
t.Fatal("error saving chart")
}
err = registryClient.PushChart(ref)
if err != nil {
t.Fatal("error pushing chart")
}
if cfg.DependingChart != nil {
c := cfg.DependingChart
dependingRef, err := ociRegistry.ParseReference(fmt.Sprintf("%s/u/ocitestuser/oci-depending-chart:1.2.3", srv.RegistryURL))
if err != nil {
t.Fatal("error parsing reference for depending chart reference")
}
err = registryClient.SaveChart(c, dependingRef)
if err != nil {
t.Fatal("error saving depending chart")
}
err = registryClient.PushChart(dependingRef)
if err != nil {
t.Fatal("error pushing depending chart")
}
}
srv.Client = registryClient
}
// NewTempServer creates a server inside of a temp dir.
//
// If the passed in string is not "", it will be treated as a shell glob, and files

Loading…
Cancel
Save