feat: add support for S3 Compatible Blob storages as StorageDriver

Signed-off-by: Björn Wenzel <bjoern-wenzel@hotmail.de>
pull/12173/head
Björn Wenzel 2 years ago
parent 03911aeab7
commit 8452e815f0
Failed to extract signature

@ -10,6 +10,10 @@ require (
github.com/Masterminds/squirrel v1.5.4 github.com/Masterminds/squirrel v1.5.4
github.com/Masterminds/vcs v1.13.3 github.com/Masterminds/vcs v1.13.3
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
github.com/aws/aws-sdk-go-v2 v1.18.0
github.com/aws/aws-sdk-go-v2/config v1.18.25
github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1
github.com/aws/smithy-go v1.13.5
github.com/containerd/containerd v1.7.0 github.com/containerd/containerd v1.7.0
github.com/cyphar/filepath-securejoin v0.2.3 github.com/cyphar/filepath-securejoin v0.2.3
github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2
@ -54,6 +58,20 @@ require (
github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.24 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect
github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd // indirect github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd // indirect

@ -82,6 +82,42 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY=
github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
github.com/aws/aws-sdk-go-v2/config v1.18.25 h1:JuYyZcnMPBiFqn87L2cRppo+rNwgah6YwD3VuyvaW6Q=
github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4=
github.com/aws/aws-sdk-go-v2/credentials v1.13.24 h1:PjiYyls3QdCrzqUN35jMWtUK1vqVZ+zLfdOa/UPFDp0=
github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 h1:jJPgroehGvjrde3XufFIJUZVK5A2L9a3KwSFgKy9n8w=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 h1:gGLG7yKaXG02/jBlg210R7VgQIotiQntNhsCFejawx8=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25 h1:AzwRi5OKKwo4QNqPf7TjeO+tK8AyOK3GVSwmRPo7/Cs=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25/go.mod h1:SUbB4wcbSEyCvqBxv/O/IBf93RbEze7U7OnoTlpPB+g=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28 h1:vGWm5vTpMr39tEZfQeDiDAMgk+5qsnvRny3FjLpnH5w=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28/go.mod h1:spfrICMD6wCAhjhzHuy6DOZZ+LAIY10UxhUmLzpJTTs=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 h1:0iKliEXAcCa2qVtRs7Ot5hItA2MsufrphbRFlz1Owxo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 h1:NbWkRxEEIRSCqxhsHQuMiTH7yo+JZW1gp8v3elSVMTQ=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2/go.mod h1:4tfW5l4IAB32VWCDEBxCRtR9T4BWy4I4kr1spr8NgZM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 h1:O+9nAy9Bb6bJFTpeNFtd9UfHbgxO1o4ZDAM9rQp5NsY=
github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1/go.mod h1:J9kLNzEiHSeGMyN7238EjJmBpCniVzFda75Gxl/NqB8=
github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 h1:UBQjaMTCKwyUYwiVnUt6toEJwGXsLBI6al083tpjJzY=
github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 h1:PkHIIJs8qvq0e5QybnZoG1K/9QTrLr9OsqCIo59jOBA=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk=
github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 h1:2DQLAKDteoEDI8zpCzqBMaZlJuoE9iTYD0gFmXVax9E=
github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8=
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@ -287,6 +323,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@ -370,6 +407,8 @@ github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=

@ -412,6 +412,17 @@ func (cfg *Configuration) Init(getter genericclioptions.RESTClientGetter, namesp
panic(fmt.Sprintf("Unable to instantiate SQL driver: %v", err)) panic(fmt.Sprintf("Unable to instantiate SQL driver: %v", err))
} }
store = storage.Init(d) store = storage.Init(d)
case "s3", "S3Driver":
d, err := driver.NewS3(
os.Getenv("HELM_DRIVER_S3_BUCKET_NAME"),
log,
namespace,
)
if err != nil {
panic(fmt.Sprintf("Unable to instantiate S3 driver: %v", err))
}
store = storage.Init(d)
default: default:
// Not sure what to do here. // Not sure what to do here.
panic("Unknown driver in HELM_DRIVER: " + helmDriver) panic("Unknown driver in HELM_DRIVER: " + helmDriver)

@ -0,0 +1,273 @@
package driver
import (
"bytes"
"context"
"errors"
"fmt"
"github.com/aws/aws-sdk-go-v2/aws"
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"helm.sh/helm/v3/pkg/release"
"io"
"net/http"
"os"
"strconv"
)
var _ Driver = (*S3Driver)(nil)
// S3DriverName is the string name of the driver.
const S3DriverName = "S3Driver"
type S3Client interface {
GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
ListObjectsV2(context.Context, *s3.ListObjectsV2Input, ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
}
// S3Driver is an implementation of the driver interface
type S3Driver struct {
bucket string
namespace string
client S3Client
Log func(string, ...interface{})
}
// NewS3 initializes a new S3Driver an implementation of the driver interface
func NewS3(bucket string, logger func(string, ...interface{}), namespace string) (*S3Driver, error) {
endpointResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
lUrl, lUrlExists := os.LookupEnv("HELM_DRIVER_S3_BUCKET_LOCATION_URL")
if service == s3.ServiceID && lUrlExists {
return aws.Endpoint{
PartitionID: "aws",
URL: lUrl,
SigningRegion: "us-east-2",
HostnameImmutable: true,
}, nil
}
// performs a fallback to the default endpoint resolver
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(context.Background(), config.WithEndpointResolverWithOptions(endpointResolver))
if err != nil {
logger("failed to load aws sdk configuration with error %v", err)
return nil, err
}
client := s3.NewFromConfig(cfg, func(options *s3.Options) {
pStyle, pStyleExists := os.LookupEnv("HELM_DRIVER_S3_USE_PATH_STYLE")
if pStyleExists {
val, err := strconv.ParseBool(pStyle)
if err == nil {
options.UsePathStyle = val
}
}
})
return &S3Driver{
client: client,
bucket: bucket,
namespace: namespace,
Log: logger,
}, nil
}
// Create creates a new release and stores it on S3. If the object already exists, ErrReleaseExists is returned.
func (s *S3Driver) Create(key string, rls *release.Release) error {
s3Key := fmt.Sprintf("%s/%s", s.namespace, key)
exists, err := s.pathAlreadyExists(s3Key)
if err != nil {
s.Log("Failed to check if release already exists with error %v", err)
return err
}
if exists {
return ErrReleaseExists
}
return s.Update(key, rls)
}
// pathAlreadyExists returns a boolean if the release at a specific path already exists
func (s *S3Driver) pathAlreadyExists(path string) (bool, error) {
input := &s3.HeadObjectInput{
Bucket: &s.bucket,
Key: &path,
}
_, err := s.client.HeadObject(context.Background(), input)
if err != nil {
var responseError *awshttp.ResponseError
if errors.As(err, &responseError) && responseError.ResponseError.HTTPStatusCode() == http.StatusNotFound {
return false, nil
}
return false, err
}
return true, nil
}
// Update updates a release by using the Create function
func (s *S3Driver) Update(key string, rls *release.Release) error {
s3Key := fmt.Sprintf("%s/%s", s.namespace, key)
r, err := encodeRelease(rls)
if err != nil {
s.Log("Failed to decode release %q with error %v", key, err)
return err
}
bodyBytes := []byte(r)
input := &s3.PutObjectInput{
Bucket: &s.bucket,
Key: &s3Key,
Body: bytes.NewReader(bodyBytes),
Metadata: map[string]string{
"name": rls.Name,
"owner": "helm",
"status": rls.Info.Status.String(),
"version": strconv.Itoa(rls.Version),
},
}
_, pErr := s.client.PutObject(context.Background(), input)
return pErr
}
// Delete loads a release and deletes it from S3
func (s *S3Driver) Delete(key string) (*release.Release, error) {
rel, err := s.Get(key)
if err != nil {
s.Log("failed to get %s with error %v", key, err)
return nil, err
}
s3Key := fmt.Sprintf("%s/%s", s.namespace, key)
input := &s3.DeleteObjectInput{
Bucket: &s.bucket,
Key: &s3Key,
}
_, delErr := s.client.DeleteObject(context.Background(), input)
if err != nil {
s.Log("failed to delete object %s with error %v", s3Key, err)
return nil, delErr
}
return rel, nil
}
// Get loads a release based on the namespace & release name
func (s *S3Driver) Get(key string) (*release.Release, error) {
s3Key := fmt.Sprintf("%s/%s", s.namespace, key)
return s.getByPath(s3Key)
}
// getByPath internal method to load a release by path consisting of the pattern namespace/release.name
func (s *S3Driver) getByPath(path string) (*release.Release, error) {
input := &s3.GetObjectInput{
Bucket: &s.bucket,
Key: &path,
}
resp, err := s.client.GetObject(context.Background(), input)
if err != nil {
var nsk *types.NoSuchKey
if errors.As(err, &nsk) {
return nil, ErrReleaseNotFound
}
s.Log("failed to get release with s3 key %s with error %v", path, err)
return nil, err
}
defer resp.Body.Close()
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
body := string(bodyBytes)
rel, err := decodeRelease(body)
if err != nil {
s.Log("failed to decode release from s3 key %s with error %v", path, err)
return nil, err
}
rel.Labels = resp.Metadata
return rel, nil
}
// List lists all releases within the S3 Bucket limited by the s.namespace parameter to limit it to a specific namespace.
func (s *S3Driver) List(filter func(*release.Release) bool) ([]*release.Release, error) {
input := &s3.ListObjectsV2Input{
Bucket: &s.bucket,
}
if len(s.namespace) != 0 {
prefix := fmt.Sprintf("%s/", s.namespace)
input.Prefix = &prefix
}
var releases []*release.Release
paginator := s3.NewListObjectsV2Paginator(s.client, input)
for paginator.HasMorePages() {
page, err := paginator.NextPage(context.Background())
if err != nil {
s.Log("failed to paginate through s3 bucket %s with error %v", s.bucket, err)
return nil, err
}
for _, obj := range page.Contents {
key := obj.Key
r, err := s.getByPath(*key)
if err != nil {
s.Log("list: failed to decode load release with key: %s: %v", key, err)
continue
}
if filter(r) {
releases = append(releases, r)
}
}
}
return releases, nil
}
// Query is an internal facade to the List function that limits the result to all objects that have the matching labels
func (s *S3Driver) Query(labels map[string]string) ([]*release.Release, error) {
releases, err := s.List(func(r *release.Release) bool {
for key, val := range labels {
if r.Labels[key] != val {
return false
}
}
return true
})
if err != nil {
s.Log("failed to query releases with error %v", err)
return nil, err
}
if len(releases) == 0 {
return nil, ErrReleaseNotFound
}
return releases, nil
}
// Name returns the name of the driver.
func (s *S3Driver) Name() string {
return S3DriverName
}

@ -0,0 +1,411 @@
package driver
import (
"bytes"
"context"
"errors"
"fmt"
"github.com/aws/aws-sdk-go-v2/aws"
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
smithyhttp "github.com/aws/smithy-go/transport/http"
"helm.sh/helm/v3/pkg/release"
"io"
"net/http"
"strconv"
"testing"
)
type mockS3Client struct {
getObjectCounter int
getObjectOverwrite func(params *s3.GetObjectInput) (*s3.GetObjectOutput, error)
putObjectCounter int
putObjectOverwrite func(params *s3.PutObjectInput) (*s3.PutObjectOutput, error)
headObjectCounter int
headObjectOverwite func(params *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
deleteObjectCounter int
deleteObjectOverwrite func(params *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error)
listObjectsV2Counter int
listObjectsV2Overwrite func(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error)
}
func (m *mockS3Client) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
m.getObjectCounter++
return m.getObjectOverwrite(params)
}
func (m *mockS3Client) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
m.putObjectCounter++
return m.putObjectOverwrite(params)
}
func (m *mockS3Client) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
m.headObjectCounter++
return m.headObjectOverwite(params)
}
func (m *mockS3Client) DeleteObject(_ context.Context, params *s3.DeleteObjectInput, _ ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) {
m.deleteObjectCounter++
return m.deleteObjectOverwrite(params)
}
func (m *mockS3Client) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) {
m.listObjectsV2Counter++
return m.listObjectsV2Overwrite(params)
}
func TestCreate(t *testing.T) {
ns := "test-namespace"
bucket := "test-bucket"
releaseName := "test-release"
key := "test-release-1"
expectedKey := fmt.Sprintf("%s/%s", ns, key)
version := 1
// Create a mock S3 client
mockClient := &mockS3Client{
putObjectOverwrite: func(params *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
if *params.Bucket != bucket {
t.Errorf("expected %s as bucket name got %s", bucket, *params.Bucket)
}
if *params.Key != expectedKey {
t.Errorf("expected %s as key but got %s", expectedKey, *params.Key)
}
if params.Metadata["name"] != releaseName {
t.Errorf("Expected metadata name key with value %s but got %s", releaseName, params.Metadata["Name"])
}
if params.Metadata["owner"] != "helm" {
t.Errorf("Expected metadata owner key with value helm but got %s", params.Metadata["owner"])
}
if params.Metadata["version"] != strconv.Itoa(version) {
t.Errorf("Expected metadata version key with value %d but got %s", version, params.Metadata["version"])
}
if params.Metadata["status"] != release.StatusPendingUpgrade.String() {
t.Errorf("Expected metadata status key with value %s but got %s", release.StatusPendingUpgrade.String(), params.Metadata["status"])
}
return nil, nil
},
headObjectOverwite: func(params *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
if *params.Bucket != bucket {
t.Errorf("expected %s as bucket name got %s", bucket, *params.Bucket)
}
if *params.Key != expectedKey {
t.Errorf("expected %s as key but got %s", expectedKey, *params.Key)
}
respErr := smithyhttp.ResponseError{
Response: &smithyhttp.Response{
Response: &http.Response{
StatusCode: 404,
},
},
}
return nil, &awshttp.ResponseError{
ResponseError: &respErr,
}
},
}
// Create an instance of S3Driver with the mock client
driver := &S3Driver{
bucket: bucket,
namespace: ns,
client: mockClient,
Log: func(string, ...interface{}) {},
}
// Set up the input for the test
rls := &release.Release{
Name: releaseName,
Version: version,
Info: &release.Info{Status: release.StatusPendingUpgrade},
}
// Call the Create function
err := driver.Create(key, rls)
// Check if the Create function returned an error
if err != nil {
t.Fatalf("Create returned an unexpected error: %v", err)
}
if mockClient.putObjectCounter != 1 {
t.Errorf("Expected PutObject to be called once but was called %d", mockClient.putObjectCounter)
}
if mockClient.headObjectCounter != 1 {
t.Errorf("Expected HeadObject to be called once but was called %d", mockClient.headObjectCounter)
}
}
func TestUpdate(t *testing.T) {
namespace := "test-namespace"
releaseName := "release"
version := 1
key := "test-key"
bucket := "test-bucket"
bucketKey := fmt.Sprintf("%s/%s", namespace, key)
mockClient := &mockS3Client{
putObjectOverwrite: func(params *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
if *params.Key != bucketKey {
t.Errorf("Expected key %s but got %s", bucketKey, *params.Key)
}
if *params.Bucket != bucket {
t.Errorf("Expected bucket %s but got %s", bucket, *params.Bucket)
}
if params.Metadata["name"] != releaseName {
t.Errorf("Expected metadata name key with value %s but got %s", releaseName, params.Metadata["Name"])
}
if params.Metadata["owner"] != "helm" {
t.Errorf("Expected metadata owner key with value helm but got %s", params.Metadata["owner"])
}
if params.Metadata["version"] != strconv.Itoa(version) {
t.Errorf("Expected metadata version key with value %d but got %s", version, params.Metadata["version"])
}
if params.Metadata["status"] != release.StatusDeployed.String() {
t.Errorf("Expected metadata status key with value %s but got %s", release.StatusDeployed.String(), params.Metadata["status"])
}
return &s3.PutObjectOutput{}, nil
},
}
// Create an instance of S3Driver with the mock client
driver := &S3Driver{
bucket: bucket,
namespace: namespace,
client: mockClient,
Log: func(string, ...interface{}) {},
}
// Set up the input for the test
rls := &release.Release{
Name: releaseName,
Version: version,
Info: &release.Info{Status: release.StatusDeployed},
}
// Call the Update function
err := driver.Update(key, rls)
// Check if the Update function returned an error
if err != nil {
t.Fatalf("Update returned an unexpected error: %v", err)
}
if mockClient.putObjectCounter != 1 {
t.Errorf("Expected PutObject to be called once but was called %d", mockClient.putObjectCounter)
}
}
func releaseToGetObjectOutput(rls *release.Release) *s3.GetObjectOutput {
b, _ := encodeRelease(rls)
bReader := bytes.NewReader([]byte(b))
readCloser := io.NopCloser(bReader)
return &s3.GetObjectOutput{
Body: readCloser,
Metadata: map[string]string{
"name": rls.Name,
"owner": "helm",
"version": strconv.Itoa(rls.Version),
"status": release.StatusDeployed.String(),
},
}
}
func TestGet(t *testing.T) {
key := "test-key"
releaseName := "test-release"
version := 1
namespace := "test-namespace"
bucket := "test-bucket"
mockClient := &mockS3Client{
getObjectOverwrite: func(params *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
if *params.Bucket != bucket {
t.Errorf("Expected bucket %s but got %s", bucket, *params.Bucket)
}
expectedKey := fmt.Sprintf("%s/%s", namespace, key)
if *params.Key != expectedKey {
t.Errorf("Expected key %s but got %s", expectedKey, *params.Key)
}
rls := &release.Release{
Name: releaseName,
Version: version,
Info: &release.Info{Status: release.StatusDeployed},
}
return releaseToGetObjectOutput(rls), nil
},
}
// Create an instance of S3Driver with the mock client
driver := &S3Driver{
bucket: bucket,
namespace: namespace,
client: mockClient,
Log: func(string, ...interface{}) {},
}
resp, err := driver.Get(key)
if err != nil {
t.Fatalf("Get returned an unexpected error %v", err)
}
if mockClient.getObjectCounter != 1 {
t.Errorf("Expected GetObject to be called once but was called %d", mockClient.getObjectCounter)
}
if resp.Name != releaseName {
t.Errorf("Expected release name %s but got %s", releaseName, resp.Name)
}
if resp.Version != version {
t.Errorf("Expected version %d but got %d", version, resp.Version)
}
if resp.Info.Status != release.StatusDeployed {
t.Errorf("Expected status %v but got %v", release.StatusDeployed, resp.Info.Status)
}
if resp.Labels["name"] != releaseName {
t.Errorf("Expected metadata name key with value %s but got %s", releaseName, resp.Labels["Name"])
}
if resp.Labels["owner"] != "helm" {
t.Errorf("Expected metadata owner key with value helm but got %s", resp.Labels["owner"])
}
if resp.Labels["version"] != strconv.Itoa(version) {
t.Errorf("Expected metadata version key with value %d but got %s", version, resp.Labels["version"])
}
if resp.Labels["status"] != release.StatusDeployed.String() {
t.Errorf("Expected metadata status key with value %s but got %s", release.StatusDeployed.String(), resp.Labels["status"])
}
}
func mockReleases() map[string]release.Release {
return map[string]release.Release{
"test-namespace/first-release-1": *releaseStub("first-release", 1, "test-namespace", release.StatusDeployed),
"test-namespace/second-release-2": *releaseStub("second-release", 2, "test-namespace", release.StatusPendingUpgrade),
"other-test-namespace/third-release-3": *releaseStub("third-release", 3, "other-test-namespace", release.StatusFailed),
}
}
func TestList(t *testing.T) {
bucket := "test-bucket"
rls := mockReleases()
mockClient := &mockS3Client{
listObjectsV2Overwrite: func(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
if input.Prefix != nil {
t.Errorf("Expected empty prefix in ListObjectsV2 but got %s", *input.Prefix)
}
if *input.Bucket != bucket {
t.Errorf("Expected bucket %s but got %s", bucket, *input.Bucket)
}
lstResult := []types.Object{}
for s, _ := range rls {
lstResult = append(lstResult, types.Object{
Key: aws.String(s),
})
}
return &s3.ListObjectsV2Output{
IsTruncated: false,
Contents: lstResult,
}, nil
},
getObjectOverwrite: func(params *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
if value, exists := rls[*params.Key]; exists {
return releaseToGetObjectOutput(&value), nil
}
t.Fatalf("GetObject was called with %s doesn't exist in releases", *params.Key)
return nil, errors.New("GetObject was called with wrong key")
},
}
driver := &S3Driver{
bucket: bucket,
namespace: "",
client: mockClient,
Log: func(string, ...interface{}) {},
}
resp, err := driver.List(func(r *release.Release) bool {
return true
})
if err != nil {
t.Fatalf("List returned an unexpected error %v", err)
}
if len(resp) != 3 {
t.Errorf("Expected 3 releases to be returned but got only %d", len(resp))
}
}
func TestQuery(t *testing.T) {
bucket := "test-bucket"
rls := mockReleases()
mockClient := &mockS3Client{
listObjectsV2Overwrite: func(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
lstResult := []types.Object{}
for s := range rls {
lstResult = append(lstResult, types.Object{
Key: aws.String(s),
})
}
return &s3.ListObjectsV2Output{
IsTruncated: false,
Contents: lstResult,
}, nil
},
getObjectOverwrite: func(params *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
if value, exists := rls[*params.Key]; exists {
return releaseToGetObjectOutput(&value), nil
}
t.Fatalf("GetObject was called with %s doesn't exist in releases", *params.Key)
return nil, errors.New("GetObject was called with wrong key")
},
}
driver := &S3Driver{
bucket: bucket,
namespace: "",
client: mockClient,
Log: func(string, ...interface{}) {},
}
resp, err := driver.Query(map[string]string{"name": "first-release"})
if err != nil {
t.Fatalf("Query returned an unexpected error %v", err)
}
if len(resp) != 1 {
t.Fatalf("Expected 1 releases to be returned but got only %d", len(resp))
}
if resp[0].Name != "first-release" {
t.Errorf("Expected the only release to be available to be named first-release but got %s", resp[0].Name)
}
}
Loading…
Cancel
Save