mirror of https://github.com/helm/helm
Merge pull request #31165 from mattfarina/content-cache
Initial addition of content based cachepull/31178/head
Matt Farina
1 month ago
committed by
GitHub
commit
0f1b410f14
@ -0,0 +1,89 @@
|
||||
/*
|
||||
Copyright The Helm Authors.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"helm.sh/helm/v4/internal/fileutil"
|
||||
)
|
||||
|
||||
// Cache describes a cache that can get and put chart data.
|
||||
// The cache key is the sha256 has of the content. sha256 is used in Helm for
|
||||
// digests in index files providing a common key for checking content.
|
||||
type Cache interface {
|
||||
// Get returns a reader for the given key.
|
||||
Get(key [sha256.Size]byte, cacheType string) (string, error)
|
||||
// Put stores the given reader for the given key.
|
||||
Put(key [sha256.Size]byte, data io.Reader, cacheType string) (string, error)
|
||||
}
|
||||
|
||||
// CacheChart specifies the content is a chart
|
||||
var CacheChart = ".chart"
|
||||
|
||||
// CacheProv specifies the content is a provenance file
|
||||
var CacheProv = ".prov"
|
||||
|
||||
// TODO: The cache assumes files because much of Helm assumes files. Convert
|
||||
// Helm to pass content around instead of file locations.
|
||||
|
||||
// DiskCache is a cache that stores data on disk.
|
||||
type DiskCache struct {
|
||||
Root string
|
||||
}
|
||||
|
||||
// Get returns a reader for the given key.
|
||||
func (c *DiskCache) Get(key [sha256.Size]byte, cacheType string) (string, error) {
|
||||
p := c.fileName(key, cacheType)
|
||||
fi, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Empty files treated as not exist because there is no content.
|
||||
if fi.Size() == 0 {
|
||||
return p, os.ErrNotExist
|
||||
}
|
||||
// directories should never happen unless something outside helm is operating
|
||||
// on this content.
|
||||
if fi.IsDir() {
|
||||
return p, errors.New("is a directory")
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// Put stores the given reader for the given key.
|
||||
// It returns the path to the stored file.
|
||||
func (c *DiskCache) Put(key [sha256.Size]byte, data io.Reader, cacheType string) (string, error) {
|
||||
// TODO: verify the key and digest of the key are the same.
|
||||
p := c.fileName(key, cacheType)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil {
|
||||
slog.Error("failed to create cache directory")
|
||||
return p, err
|
||||
}
|
||||
return p, fileutil.AtomicWriteFile(p, data, 0644)
|
||||
}
|
||||
|
||||
// fileName generates the filename in a structured manner where the first part is the
|
||||
// directory and the full hash is the filename.
|
||||
func (c *DiskCache) fileName(id [sha256.Size]byte, cacheType string) string {
|
||||
return filepath.Join(c.Root, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+cacheType)
|
||||
}
|
||||
@ -0,0 +1,122 @@
|
||||
/*
|
||||
Copyright The Helm Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// compiler check to ensure DiskCache implements the Cache interface.
|
||||
var _ Cache = (*DiskCache)(nil)
|
||||
|
||||
func TestDiskCache_PutAndGet(t *testing.T) {
|
||||
// Setup a temporary directory for the cache
|
||||
tmpDir := t.TempDir()
|
||||
cache := &DiskCache{Root: tmpDir}
|
||||
|
||||
// Test data
|
||||
content := []byte("hello world")
|
||||
key := sha256.Sum256(content)
|
||||
|
||||
// --- Test case 1: Put and Get a regular file (prov=false) ---
|
||||
t.Run("PutAndGetTgz", func(t *testing.T) {
|
||||
// Put the data into the cache
|
||||
path, err := cache.Put(key, bytes.NewReader(content), CacheChart)
|
||||
require.NoError(t, err, "Put should not return an error")
|
||||
|
||||
// Verify the file exists at the returned path
|
||||
_, err = os.Stat(path)
|
||||
require.NoError(t, err, "File should exist after Put")
|
||||
|
||||
// Get the file from the cache
|
||||
retrievedPath, err := cache.Get(key, CacheChart)
|
||||
require.NoError(t, err, "Get should not return an error for existing file")
|
||||
assert.Equal(t, path, retrievedPath, "Get should return the same path as Put")
|
||||
|
||||
// Verify content
|
||||
data, err := os.ReadFile(retrievedPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, data, "Content of retrieved file should match original content")
|
||||
})
|
||||
|
||||
// --- Test case 2: Put and Get a provenance file (prov=true) ---
|
||||
t.Run("PutAndGetProv", func(t *testing.T) {
|
||||
provContent := []byte("provenance data")
|
||||
provKey := sha256.Sum256(provContent)
|
||||
|
||||
path, err := cache.Put(provKey, bytes.NewReader(provContent), CacheProv)
|
||||
require.NoError(t, err)
|
||||
|
||||
retrievedPath, err := cache.Get(provKey, CacheProv)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, path, retrievedPath)
|
||||
|
||||
data, err := os.ReadFile(retrievedPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, provContent, data)
|
||||
})
|
||||
|
||||
// --- Test case 3: Get a non-existent file ---
|
||||
t.Run("GetNonExistent", func(t *testing.T) {
|
||||
nonExistentKey := sha256.Sum256([]byte("does not exist"))
|
||||
_, err := cache.Get(nonExistentKey, CacheChart)
|
||||
assert.ErrorIs(t, err, os.ErrNotExist, "Get for a non-existent key should return os.ErrNotExist")
|
||||
})
|
||||
|
||||
// --- Test case 4: Put an empty file ---
|
||||
t.Run("PutEmptyFile", func(t *testing.T) {
|
||||
emptyContent := []byte{}
|
||||
emptyKey := sha256.Sum256(emptyContent)
|
||||
|
||||
path, err := cache.Put(emptyKey, bytes.NewReader(emptyContent), CacheChart)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get should return ErrNotExist for empty files
|
||||
_, err = cache.Get(emptyKey, CacheChart)
|
||||
assert.ErrorIs(t, err, os.ErrNotExist, "Get for an empty file should return os.ErrNotExist")
|
||||
|
||||
// But the file should exist
|
||||
_, err = os.Stat(path)
|
||||
require.NoError(t, err, "Empty file should still exist on disk")
|
||||
})
|
||||
|
||||
// --- Test case 5: Get a directory ---
|
||||
t.Run("GetDirectory", func(t *testing.T) {
|
||||
dirKey := sha256.Sum256([]byte("i am a directory"))
|
||||
dirPath := cache.fileName(dirKey, CacheChart)
|
||||
err := os.MkdirAll(dirPath, 0755)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = cache.Get(dirKey, CacheChart)
|
||||
assert.EqualError(t, err, "is a directory")
|
||||
})
|
||||
}
|
||||
|
||||
func TestDiskCache_fileName(t *testing.T) {
|
||||
cache := &DiskCache{Root: "/tmp/cache"}
|
||||
key := sha256.Sum256([]byte("some data"))
|
||||
|
||||
assert.Equal(t, filepath.Join("/tmp/cache", "13", "1307990e6ba5ca145eb35e99182a9bec46531bc54ddf656a602c780fa0240dee.chart"), cache.fileName(key, CacheChart))
|
||||
assert.Equal(t, filepath.Join("/tmp/cache", "13", "1307990e6ba5ca145eb35e99182a9bec46531bc54ddf656a602c780fa0240dee.prov"), cache.fileName(key, CacheProv))
|
||||
}
|
||||
Loading…
Reference in new issue