mirror of https://github.com/helm/helm
Merge pull request #31165 from mattfarina/content-cache
Initial addition of content based cachepull/31178/head
Matt Farina
4 weeks ago
committed by
GitHub
commit
0f1b410f14
@ -0,0 +1,89 @@
|
|||||||
|
/*
|
||||||
|
Copyright The Helm Authors.
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package downloader
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"helm.sh/helm/v4/internal/fileutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Cache describes a cache that can get and put chart data.
|
||||||
|
// The cache key is the sha256 has of the content. sha256 is used in Helm for
|
||||||
|
// digests in index files providing a common key for checking content.
|
||||||
|
type Cache interface {
|
||||||
|
// Get returns a reader for the given key.
|
||||||
|
Get(key [sha256.Size]byte, cacheType string) (string, error)
|
||||||
|
// Put stores the given reader for the given key.
|
||||||
|
Put(key [sha256.Size]byte, data io.Reader, cacheType string) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CacheChart specifies the content is a chart
|
||||||
|
var CacheChart = ".chart"
|
||||||
|
|
||||||
|
// CacheProv specifies the content is a provenance file
|
||||||
|
var CacheProv = ".prov"
|
||||||
|
|
||||||
|
// TODO: The cache assumes files because much of Helm assumes files. Convert
|
||||||
|
// Helm to pass content around instead of file locations.
|
||||||
|
|
||||||
|
// DiskCache is a cache that stores data on disk.
|
||||||
|
type DiskCache struct {
|
||||||
|
Root string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns a reader for the given key.
|
||||||
|
func (c *DiskCache) Get(key [sha256.Size]byte, cacheType string) (string, error) {
|
||||||
|
p := c.fileName(key, cacheType)
|
||||||
|
fi, err := os.Stat(p)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
// Empty files treated as not exist because there is no content.
|
||||||
|
if fi.Size() == 0 {
|
||||||
|
return p, os.ErrNotExist
|
||||||
|
}
|
||||||
|
// directories should never happen unless something outside helm is operating
|
||||||
|
// on this content.
|
||||||
|
if fi.IsDir() {
|
||||||
|
return p, errors.New("is a directory")
|
||||||
|
}
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put stores the given reader for the given key.
|
||||||
|
// It returns the path to the stored file.
|
||||||
|
func (c *DiskCache) Put(key [sha256.Size]byte, data io.Reader, cacheType string) (string, error) {
|
||||||
|
// TODO: verify the key and digest of the key are the same.
|
||||||
|
p := c.fileName(key, cacheType)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil {
|
||||||
|
slog.Error("failed to create cache directory")
|
||||||
|
return p, err
|
||||||
|
}
|
||||||
|
return p, fileutil.AtomicWriteFile(p, data, 0644)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fileName generates the filename in a structured manner where the first part is the
|
||||||
|
// directory and the full hash is the filename.
|
||||||
|
func (c *DiskCache) fileName(id [sha256.Size]byte, cacheType string) string {
|
||||||
|
return filepath.Join(c.Root, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+cacheType)
|
||||||
|
}
|
||||||
@ -0,0 +1,122 @@
|
|||||||
|
/*
|
||||||
|
Copyright The Helm Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package downloader
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/sha256"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// compiler check to ensure DiskCache implements the Cache interface.
|
||||||
|
var _ Cache = (*DiskCache)(nil)
|
||||||
|
|
||||||
|
func TestDiskCache_PutAndGet(t *testing.T) {
|
||||||
|
// Setup a temporary directory for the cache
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
cache := &DiskCache{Root: tmpDir}
|
||||||
|
|
||||||
|
// Test data
|
||||||
|
content := []byte("hello world")
|
||||||
|
key := sha256.Sum256(content)
|
||||||
|
|
||||||
|
// --- Test case 1: Put and Get a regular file (prov=false) ---
|
||||||
|
t.Run("PutAndGetTgz", func(t *testing.T) {
|
||||||
|
// Put the data into the cache
|
||||||
|
path, err := cache.Put(key, bytes.NewReader(content), CacheChart)
|
||||||
|
require.NoError(t, err, "Put should not return an error")
|
||||||
|
|
||||||
|
// Verify the file exists at the returned path
|
||||||
|
_, err = os.Stat(path)
|
||||||
|
require.NoError(t, err, "File should exist after Put")
|
||||||
|
|
||||||
|
// Get the file from the cache
|
||||||
|
retrievedPath, err := cache.Get(key, CacheChart)
|
||||||
|
require.NoError(t, err, "Get should not return an error for existing file")
|
||||||
|
assert.Equal(t, path, retrievedPath, "Get should return the same path as Put")
|
||||||
|
|
||||||
|
// Verify content
|
||||||
|
data, err := os.ReadFile(retrievedPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, content, data, "Content of retrieved file should match original content")
|
||||||
|
})
|
||||||
|
|
||||||
|
// --- Test case 2: Put and Get a provenance file (prov=true) ---
|
||||||
|
t.Run("PutAndGetProv", func(t *testing.T) {
|
||||||
|
provContent := []byte("provenance data")
|
||||||
|
provKey := sha256.Sum256(provContent)
|
||||||
|
|
||||||
|
path, err := cache.Put(provKey, bytes.NewReader(provContent), CacheProv)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
retrievedPath, err := cache.Get(provKey, CacheProv)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, path, retrievedPath)
|
||||||
|
|
||||||
|
data, err := os.ReadFile(retrievedPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, provContent, data)
|
||||||
|
})
|
||||||
|
|
||||||
|
// --- Test case 3: Get a non-existent file ---
|
||||||
|
t.Run("GetNonExistent", func(t *testing.T) {
|
||||||
|
nonExistentKey := sha256.Sum256([]byte("does not exist"))
|
||||||
|
_, err := cache.Get(nonExistentKey, CacheChart)
|
||||||
|
assert.ErrorIs(t, err, os.ErrNotExist, "Get for a non-existent key should return os.ErrNotExist")
|
||||||
|
})
|
||||||
|
|
||||||
|
// --- Test case 4: Put an empty file ---
|
||||||
|
t.Run("PutEmptyFile", func(t *testing.T) {
|
||||||
|
emptyContent := []byte{}
|
||||||
|
emptyKey := sha256.Sum256(emptyContent)
|
||||||
|
|
||||||
|
path, err := cache.Put(emptyKey, bytes.NewReader(emptyContent), CacheChart)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Get should return ErrNotExist for empty files
|
||||||
|
_, err = cache.Get(emptyKey, CacheChart)
|
||||||
|
assert.ErrorIs(t, err, os.ErrNotExist, "Get for an empty file should return os.ErrNotExist")
|
||||||
|
|
||||||
|
// But the file should exist
|
||||||
|
_, err = os.Stat(path)
|
||||||
|
require.NoError(t, err, "Empty file should still exist on disk")
|
||||||
|
})
|
||||||
|
|
||||||
|
// --- Test case 5: Get a directory ---
|
||||||
|
t.Run("GetDirectory", func(t *testing.T) {
|
||||||
|
dirKey := sha256.Sum256([]byte("i am a directory"))
|
||||||
|
dirPath := cache.fileName(dirKey, CacheChart)
|
||||||
|
err := os.MkdirAll(dirPath, 0755)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = cache.Get(dirKey, CacheChart)
|
||||||
|
assert.EqualError(t, err, "is a directory")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiskCache_fileName(t *testing.T) {
|
||||||
|
cache := &DiskCache{Root: "/tmp/cache"}
|
||||||
|
key := sha256.Sum256([]byte("some data"))
|
||||||
|
|
||||||
|
assert.Equal(t, filepath.Join("/tmp/cache", "13", "1307990e6ba5ca145eb35e99182a9bec46531bc54ddf656a602c780fa0240dee.chart"), cache.fileName(key, CacheChart))
|
||||||
|
assert.Equal(t, filepath.Join("/tmp/cache", "13", "1307990e6ba5ca145eb35e99182a9bec46531bc54ddf656a602c780fa0240dee.prov"), cache.fileName(key, CacheProv))
|
||||||
|
}
|
||||||
Loading…
Reference in new issue