feat(pkg) Setting settings.RepositoryCache="off" does not save charts in fs

Setting the value of settings.RepositoryCache to "off" does not store the
charts in the filesystem. This will download chart to a LRU cache
directly for use with provenance verification done in memory as well.

Signed-off-by: Vibhav Bobade <vibhav.bobde@gmail.com>

cmd/helm/install.go

@@ -19,6 +19,7 @@ package main
 import (
 	"fmt"
 	"io"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -163,13 +164,13 @@ func runInstall(args []string, client *action.Install, valueOpts *values.Options
 		client.Version = ">0.0.0-0"
 	}
 
-	name, chart, err := client.NameAndChart(args)
+	name, schart, err := client.NameAndChart(args)
 	if err != nil {
 		return nil, err
 	}
 	client.ReleaseName = name
 
-	cp, err := client.ChartPathOptions.LocateChart(chart, settings)
+	cp, err := client.ChartPathOptions.LocateChart(schart, settings)
 	if err != nil {
 		return nil, err
 	}
@@ -183,7 +184,12 @@ func runInstall(args []string, client *action.Install, valueOpts *values.Options
 	}
 
 	// Check chart dependencies to make sure all are present in /charts
-	chartRequested, err := loader.Load(cp)
+	var chartRequested *chart.Chart
+	if settings.RepositoryCache == "off" {
+		chartRequested, err = downloader.GetChart(schart, strings.TrimSpace(client.ChartPathOptions.Version))
+	} else {
+		chartRequested, err = loader.Load(cp)
+	}
 	if err != nil {
 		return nil, err
 	}

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/gosuri/uitable v0.0.4
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/lib/pq v1.3.0
+	github.com/hashicorp/golang-lru v0.5.1
 	github.com/mattn/go-shellwords v1.0.10
 	github.com/mitchellh/copystructure v1.0.0
 	github.com/opencontainers/go-digest v1.0.0-rc1

pkg/action/install.go

@@ -620,7 +620,6 @@ OUTER:
 func (c *ChartPathOptions) LocateChart(name string, settings *cli.EnvSettings) (string, error) {
 	name = strings.TrimSpace(name)
 	version := strings.TrimSpace(c.Version)
-
 	if _, err := os.Stat(name); err == nil {
 		abs, err := filepath.Abs(name)
 		if err != nil {
@@ -660,20 +659,24 @@ func (c *ChartPathOptions) LocateChart(name string, settings *cli.EnvSettings) (
 		name = chartURL
 	}
 
-	if err := os.MkdirAll(settings.RepositoryCache, 0755); err != nil {
+	// If HELM_REPOSITORY_CACHE has an "off" value, the chart won't be downloaded
+	if err := os.MkdirAll(settings.RepositoryCache, 0755); err != nil && settings.RepositoryCache != "off" {
 		return "", err
 	}
 
-	filename, _, err := dl.DownloadTo(name, version, settings.RepositoryCache)
+	chartLocation, _, err := dl.DownloadTo(name, version, settings.RepositoryCache)
 	if err == nil {
-		lname, err := filepath.Abs(filename)
+		if settings.RepositoryCache != "off" {
+			return chartLocation, err
+		}
+		lname, err := filepath.Abs(chartLocation)
 		if err != nil {
-			return filename, err
+			return chartLocation, err
 		}
 		return lname, nil
 	} else if settings.Debug {
-		return filename, err
+		return chartLocation, err
 	}
 
-	return filename, errors.Errorf("failed to download %q (hint: running `helm repo update` may help)", name)
+	return chartLocation, errors.Errorf("failed to download %q (hint: running `helm repo update` may help)", name)
 }

pkg/downloader/chart_downloader.go

@@ -16,6 +16,7 @@ limitations under the License.
 package downloader
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"net/url"
@@ -23,6 +24,11 @@ import (
 	"path/filepath"
 	"strings"
 
+	lru "github.com/hashicorp/golang-lru"
+
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/chart/loader"
+
 	"github.com/pkg/errors"
 
 	"helm.sh/helm/v3/internal/fileutil"
@@ -72,6 +78,16 @@ type ChartDownloader struct {
 	RepositoryCache  string
 }
 
+type inMemoryChart struct {
+	ChartData              *bytes.Buffer
+	ProvenanceVerification *provenance.Verification
+}
+
+// inMemoryCharts maps version to a particular chart
+type inMemoryCharts map[string]inMemoryChart
+
+var inMemoryCache, _ = lru.New(128)
+
 // DownloadTo retrieves a chart. Depending on the settings, it may also download a provenance file.
 //
 // If Verify is set to VerifyNever, the verification will be nil.
@@ -94,43 +110,74 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven
 		return "", nil, err
 	}
 
-	data, err := g.Get(u.String(), c.Options...)
+	chartData, err := g.Get(u.String(), c.Options...)
 	if err != nil {
 		return "", nil, err
 	}
 
-	name := filepath.Base(u.Path)
-	destfile := filepath.Join(dest, name)
-	if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil {
-		return destfile, nil, err
-	}
-
 	// If provenance is requested, verify it.
 	ver := &provenance.Verification{}
+	chartRef := ""
+	if dest == "off" {
+		chartRef = ref
+	} else {
+		filename := filepath.Base(u.Path)
+		destfile := filepath.Join(dest, filename)
+		if err := fileutil.AtomicWriteFile(destfile, chartData, 0644); err != nil {
+			return destfile, nil, err
+		}
+		chartRef = destfile
+	}
 	if c.Verify > VerifyNever {
-		body, err := g.Get(u.String() + ".prov")
+		provData, err := g.Get(u.String() + ".prov")
 		if err != nil {
 			if c.Verify == VerifyAlways {
-				return destfile, ver, errors.Errorf("failed to fetch provenance %q", u.String()+".prov")
+				return chartRef, ver, errors.Errorf("failed to fetch provenance %q", u.String()+".prov")
 			}
 			fmt.Fprintf(c.Out, "WARNING: Verification not found for %s: %s\n", ref, err)
-			return destfile, ver, nil
+			return chartRef, ver, nil
 		}
-		provfile := destfile + ".prov"
+
-		if err := fileutil.AtomicWriteFile(provfile, body, 0644); err != nil {
+		if dest == "off" {
-			return destfile, nil, err
+			if c.Verify != VerifyLater {
+				ver, err = verifyInMemoryChart(chartRef, c.Keyring, chartData, provData)
+				if err != nil {
+					// Fail always in this case, since it means the verification step
+					// failed.
+					return chartRef, ver, err
+				}
+			}
+		} else {
+			provfile := chartRef + ".prov"
+			if err := fileutil.AtomicWriteFile(provfile, provData, 0644); err != nil {
+				return chartRef, nil, err
 			}
 
 			if c.Verify != VerifyLater {
-			ver, err = VerifyChart(destfile, c.Keyring)
+				ver, err = VerifyChart(chartRef, c.Keyring)
 				if err != nil {
 					// Fail always in this case, since it means the verification step
 					// failed.
-				return destfile, ver, err
+					return chartRef, ver, err
 				}
 			}
 		}
-	return destfile, ver, nil
+	}
+
+	if dest == "off" {
+		inMemoryCache.ContainsOrAdd(chartRef, make(inMemoryCharts))
+		tempCharts, ok := inMemoryCache.Get(chartRef)
+
+		if !ok {
+			return chartRef, ver, errors.New(fmt.Sprintf("error during retrival of chart list %s from Cache", ref))
+		}
+		tempCharts.(inMemoryCharts)[version] = inMemoryChart{
+			ChartData:              chartData,
+			ProvenanceVerification: ver,
+		}
+		inMemoryCache.Add(chartRef, tempCharts)
+	}
+	return chartRef, ver, nil
 }
 
 // ResolveChartVersion resolves a chart reference to a URL.
@@ -154,6 +201,10 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, er
 	}
 	c.Options = append(c.Options, getter.WithURL(ref))
 
+	if c.RepositoryCache == "off" {
+		return u, nil
+	}
+
 	rf, err := loadRepoConfig(c.RepositoryConfig)
 	if err != nil {
 		return u, err
@@ -293,6 +344,19 @@ func VerifyChart(path, keyring string) (*provenance.Verification, error) {
 	return sig.Verify(path, provfile)
 }
 
+// verifyInMemoryChart takes key to a chart in the the inMemoryCache and a keyring,
+// and verifies the chart.
+//
+// It assumes that a chart data is accompanied by a embedded provenance file whose
+// name is the archive file name plus the ".prov" extension.
+func verifyInMemoryChart(ref, keyring string, chartData, provData *bytes.Buffer) (*provenance.Verification, error) {
+	sig, err := provenance.NewFromKeyring(keyring, "")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to load keyring")
+	}
+	return sig.VerifyFromData(ref, chartData, provData)
+}
+
 // isTar tests whether the given file is a tar file.
 //
 // Currently, this simply checks extension, since a subsequent function will
@@ -367,3 +431,12 @@ func loadRepoConfig(file string) (*repo.File, error) {
 	}
 	return r, nil
 }
+
+func GetChart(chart string, version string) (*chart.Chart, error) {
+	chartList, _ := inMemoryCache.Get(chart)
+	if chartList != nil {
+		chartData := chartList.(inMemoryCharts)[version].ChartData
+		return loader.LoadArchive(chartData)
+	}
+	return nil, errors.New("Chart not found in cache")
+}

pkg/downloader/chart_downloader_test.go

@@ -322,6 +322,84 @@ func TestDownloadTo_VerifyLater(t *testing.T) {
 	}
 }
 
+func TestDownloadToInMemoryCache(t *testing.T) {
+	// Set up a fake repo with basic auth enabled
+	srv, err := repotest.NewTempServer("testdata/*.tgz*")
+	srv.Stop()
+	if err != nil {
+		t.Fatal(err)
+	}
+	srv.WithMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		username, password, ok := r.BasicAuth()
+		if !ok || username != "username" || password != "password" {
+			t.Errorf("Expected request to use basic auth and for username == 'username' and password == 'password', got '%v', '%s', '%s'", ok, username, password)
+		}
+	}))
+	srv.Start()
+	defer srv.Stop()
+	if err := srv.CreateIndex(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := srv.LinkIndices(); err != nil {
+		t.Fatal(err)
+	}
+
+	c := ChartDownloader{
+		Out:              os.Stderr,
+		Verify:           VerifyAlways,
+		Keyring:          "testdata/helm-test-key.pub",
+		RepositoryConfig: repoConfig,
+		RepositoryCache:  repoCache,
+		Getters: getter.All(&cli.EnvSettings{
+			RepositoryConfig: repoConfig,
+			RepositoryCache:  repoCache,
+		}),
+		Options: []getter.Option{
+			getter.WithBasicAuth("username", "password"),
+		},
+	}
+	cname := "/signtest-0.1.0.tgz"
+	_, v, err := c.DownloadTo(srv.URL()+cname, "", "off")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if v.FileHash == "" {
+		t.Error("File hash was empty, but verification is required.")
+	}
+}
+
+func TestDownloadToInMemoryCache_VerifyLater(t *testing.T) {
+	defer ensure.HelmHome(t)()
+
+	// Set up a fake repo
+	srv, err := repotest.NewTempServer("testdata/*.tgz*")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer srv.Stop()
+	if err := srv.LinkIndices(); err != nil {
+		t.Fatal(err)
+	}
+
+	c := ChartDownloader{
+		Out:              os.Stderr,
+		Verify:           VerifyLater,
+		RepositoryConfig: repoConfig,
+		RepositoryCache:  repoCache,
+		Getters: getter.All(&cli.EnvSettings{
+			RepositoryConfig: repoConfig,
+			RepositoryCache:  repoCache,
+		}),
+	}
+	cname := "/signtest-0.1.0.tgz"
+	_, _, err = c.DownloadTo(srv.URL()+cname, "", "off")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestScanReposForURL(t *testing.T) {
 	c := ChartDownloader{
 		Out:              os.Stderr,

pkg/provenance/sign.go

@@ -229,7 +229,7 @@ func (s *Signatory) ClearSign(chartpath string) (string, error) {
 	return out.String(), err
 }
 
-// Verify checks a signature and verifies that it is legit for a chart.
+// Verify checks a signature and verifies that it is legit for a chart (stored in the filesystem)
 func (s *Signatory) Verify(chartpath, sigpath string) (*Verification, error) {
 	ver := &Verification{}
 	for _, fname := range []string{chartpath, sigpath} {
@@ -240,8 +240,31 @@ func (s *Signatory) Verify(chartpath, sigpath string) (*Verification, error) {
 		}
 	}
 
+	// Get sigData
+	sigData, err := ioutil.ReadFile(sigpath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get chartData
+	chartData, err := ioutil.ReadFile(chartpath)
+	if err != nil {
+		return nil, err
+	}
+
+	ver, nil := s.VerifyFromData(chartpath, bytes.NewBuffer(chartData), bytes.NewBuffer(sigData))
+
+	// TODO: when image signing is added, verify that here.
+
+	return ver, nil
+}
+
+// VerifyFromData checks the signature and verifies that it is legit for a chart (not stored in the filesystem)
+func (s *Signatory) VerifyFromData(chartRef string, chartData, sigData *bytes.Buffer) (*Verification, error) {
+	ver := &Verification{}
+
 	// First verify the signature
-	sig, err := s.decodeSignature(sigpath)
+	sig, err := s.decodeSignatureFromData(sigData.Bytes())
 	if err != nil {
 		return ver, errors.Wrap(err, "failed to decode signature")
 	}
@@ -253,28 +276,37 @@ func (s *Signatory) Verify(chartpath, sigpath string) (*Verification, error) {
 	ver.SignedBy = by
 
 	// Second, verify the hash of the tarball.
-	sum, err := DigestFile(chartpath)
+	sum, err := Digest(chartData)
 	if err != nil {
 		return ver, err
 	}
+
+	verification, err := s.verifySum(sum, chartRef, sig, ver)
+	if err != nil {
+		return verification, err
+	}
+
+	// TODO: when image signing is added, verify that here.
+
+	return ver, nil
+}
+
+func (s *Signatory) verifySum(sum string, chartRef string, sig *clearsign.Block, ver *Verification) (*Verification, error) {
 	_, sums, err := parseMessageBlock(sig.Plaintext)
 	if err != nil {
 		return ver, err
 	}
 
 	sum = "sha256:" + sum
-	basename := filepath.Base(chartpath)
+	basename := filepath.Base(chartRef)
 	if sha, ok := sums.Files[basename]; !ok {
-		return ver, errors.Errorf("provenance does not contain a SHA for a file named %q", basename)
+		return ver, errors.Errorf("provenance does not contain a SHA for reference named %q", basename)
 	} else if sha != sum {
 		return ver, errors.Errorf("sha256 sum does not match for %s: %q != %q", basename, sha, sum)
 	}
 	ver.FileHash = sum
 	ver.FileName = basename
-
+	return nil, nil
-	// TODO: when image signing is added, verify that here.
-
-	return ver, nil
 }
 
 func (s *Signatory) decodeSignature(filename string) (*clearsign.Block, error) {
@@ -283,6 +315,10 @@ func (s *Signatory) decodeSignature(filename string) (*clearsign.Block, error) {
 		return nil, err
 	}
 
+	return s.decodeSignatureFromData(data)
+}
+
+func (s *Signatory) decodeSignatureFromData(data []byte) (*clearsign.Block, error) {
 	block, _ := clearsign.Decode(data)
 	if block == nil {
 		// There was no sig in the file.