From e04b58cdf7e200931a09d7b3ac9355d5bd8d27de Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Wed, 27 May 2020 17:36:52 -0500 Subject: [PATCH] Cache chart layers without timestamp This commit allows for the creation of a tar.gz archive of a chart without a timestamp. Doing so ensures that charts with identical content will have identical digests, which is important for storage in a registry. Effort was made to ensure the functionality of `helm package` was not affected. Additionally, a unit test for the registry cache's StoreReference function, to help track this bug. Resolves #8212 Signed-off-by: Peter Engelbert --- internal/experimental/registry/cache.go | 2 +- pkg/action/chart_save_test.go | 46 +++++++++++++++++++++++ pkg/chartutil/save.go | 50 +++++++++++++++++++------ 3 files changed, 85 insertions(+), 13 deletions(-) diff --git a/internal/experimental/registry/cache.go b/internal/experimental/registry/cache.go index fbd62562a..2be1214f5 100644 --- a/internal/experimental/registry/cache.go +++ b/internal/experimental/registry/cache.go @@ -288,7 +288,7 @@ func (cache *Cache) saveChartConfig(ch *chart.Chart) (*ocispec.Descriptor, bool, func (cache *Cache) saveChartContentLayer(ch *chart.Chart) (*ocispec.Descriptor, bool, error) { destDir := filepath.Join(cache.rootDir, ".build") os.MkdirAll(destDir, 0755) - tmpFile, err := chartutil.Save(ch, destDir) + tmpFile, err := chartutil.Save(ch, destDir, chartutil.WithoutTimestamp()) defer os.Remove(tmpFile) if err != nil { return nil, false, errors.Wrap(err, "failed to save") diff --git a/pkg/action/chart_save_test.go b/pkg/action/chart_save_test.go index 4fd991a4e..53202a216 100644 --- a/pkg/action/chart_save_test.go +++ b/pkg/action/chart_save_test.go @@ -19,6 +19,7 @@ package action import ( "io/ioutil" "testing" + "time" "helm.sh/helm/v3/internal/experimental/registry" ) @@ -68,3 +69,48 @@ func TestChartSave(t *testing.T) { t.Error(err) } } + +func TestIdenticalDigests(t *testing.T) { + ref1String := "localhost:5000/test1:0.2.0" + ref2String := "localhost:5000/test2:0.2.0" + + tmpDir, err := ioutil.TempDir("", "helm-digest-test") + if err != nil { + t.Error(err) + } + + cache, err := registry.NewCache( + registry.CacheOptRoot(tmpDir), + ) + if err != nil { + t.Error(err) + } + + input := buildChart() + + ref1, err := registry.ParseReference(ref1String) + if err != nil { + t.Error(err) + } + + ref2, err := registry.ParseReference(ref2String) + if err != nil { + t.Error(err) + } + + sum1, err := cache.StoreReference(ref1, input) + if err != nil { + t.Error(err) + } + + time.Sleep(1 * time.Second) + + sum2, err := cache.StoreReference(ref2, input) + if err != nil { + t.Error(err) + } + + if a, b := string(sum1.Digest), string(sum2.Digest); a != b { + t.Fatalf("digest %s and digest %s do not match", a, b) + } +} diff --git a/pkg/chartutil/save.go b/pkg/chartutil/save.go index 2ce4eddaf..8e34367dd 100644 --- a/pkg/chartutil/save.go +++ b/pkg/chartutil/save.go @@ -31,6 +31,22 @@ import ( "helm.sh/helm/v3/pkg/chart" ) +type ( + saveConfig struct { + noTimestamp bool + } + + // SaveOpt is an option for modifying the behavior of Save + SaveOpt func(*saveConfig) +) + +// WithoutTimestamp provides the ability to save tar archives without a timestamp +func WithoutTimestamp() SaveOpt { + return func(cfg *saveConfig) { + cfg.noTimestamp = true + } +} + var headerBytes = []byte("+aHR0cHM6Ly95b3V0dS5iZS96OVV6MWljandyTQo=") // SaveDir saves a chart as files in a directory. @@ -99,7 +115,12 @@ func SaveDir(c *chart.Chart, dest string) error { // will generate /foo/bar-1.0.0.tgz. // // This returns the absolute path to the chart archive file. -func Save(c *chart.Chart, outDir string) (string, error) { +func Save(c *chart.Chart, outDir string, saveOpts ...SaveOpt) (string, error) { + config := new(saveConfig) + for _, fn := range saveOpts { + fn(config) + } + if err := c.Validate(); err != nil { return "", errors.Wrap(err, "chart validation") } @@ -141,14 +162,19 @@ func Save(c *chart.Chart, outDir string) (string, error) { } }() - if err := writeTarContents(twriter, c, ""); err != nil { + var timestamp time.Time + if !config.noTimestamp { + timestamp = time.Now() + } + + if err := writeTarContents(twriter, c, "", timestamp); err != nil { rollback = true return filename, err } return filename, nil } -func writeTarContents(out *tar.Writer, c *chart.Chart, prefix string) error { +func writeTarContents(out *tar.Writer, c *chart.Chart, prefix string, timestamp time.Time) error { base := filepath.Join(prefix, c.Name()) // Pull out the dependencies of a v1 Chart, since there's no way @@ -165,7 +191,7 @@ func writeTarContents(out *tar.Writer, c *chart.Chart, prefix string) error { if err != nil { return err } - if err := writeToTar(out, filepath.Join(base, ChartfileName), cdata); err != nil { + if err := writeToTar(out, filepath.Join(base, ChartfileName), cdata, timestamp); err != nil { return err } @@ -177,7 +203,7 @@ func writeTarContents(out *tar.Writer, c *chart.Chart, prefix string) error { if err != nil { return err } - if err := writeToTar(out, filepath.Join(base, "Chart.lock"), ldata); err != nil { + if err := writeToTar(out, filepath.Join(base, "Chart.lock"), ldata, timestamp); err != nil { return err } } @@ -186,7 +212,7 @@ func writeTarContents(out *tar.Writer, c *chart.Chart, prefix string) error { // Save values.yaml for _, f := range c.Raw { if f.Name == ValuesfileName { - if err := writeToTar(out, filepath.Join(base, ValuesfileName), f.Data); err != nil { + if err := writeToTar(out, filepath.Join(base, ValuesfileName), f.Data, timestamp); err != nil { return err } } @@ -197,7 +223,7 @@ func writeTarContents(out *tar.Writer, c *chart.Chart, prefix string) error { if !json.Valid(c.Schema) { return errors.New("Invalid JSON in " + SchemafileName) } - if err := writeToTar(out, filepath.Join(base, SchemafileName), c.Schema); err != nil { + if err := writeToTar(out, filepath.Join(base, SchemafileName), c.Schema, timestamp); err != nil { return err } } @@ -205,7 +231,7 @@ func writeTarContents(out *tar.Writer, c *chart.Chart, prefix string) error { // Save templates for _, f := range c.Templates { n := filepath.Join(base, f.Name) - if err := writeToTar(out, n, f.Data); err != nil { + if err := writeToTar(out, n, f.Data, timestamp); err != nil { return err } } @@ -213,14 +239,14 @@ func writeTarContents(out *tar.Writer, c *chart.Chart, prefix string) error { // Save files for _, f := range c.Files { n := filepath.Join(base, f.Name) - if err := writeToTar(out, n, f.Data); err != nil { + if err := writeToTar(out, n, f.Data, timestamp); err != nil { return err } } // Save dependencies for _, dep := range c.Dependencies() { - if err := writeTarContents(out, dep, filepath.Join(base, ChartsDir)); err != nil { + if err := writeTarContents(out, dep, filepath.Join(base, ChartsDir), timestamp); err != nil { return err } } @@ -228,13 +254,13 @@ func writeTarContents(out *tar.Writer, c *chart.Chart, prefix string) error { } // writeToTar writes a single file to a tar archive. -func writeToTar(out *tar.Writer, name string, body []byte) error { +func writeToTar(out *tar.Writer, name string, body []byte, timestamp time.Time) error { // TODO: Do we need to create dummy parent directory names if none exist? h := &tar.Header{ Name: filepath.ToSlash(name), Mode: 0644, Size: int64(len(body)), - ModTime: time.Now(), + ModTime: timestamp, } if err := out.WriteHeader(h); err != nil { return err