ref(internal/experimental/registry): pkg refactor (#6205)

No more magic separating the metadata from chart tarball - charts are
pushed to registry as a single tarball layer with Chart.yaml in tact.

No more fragile custom symlink chart storage, now following
the OCI Image Layout Specification for chart filesystem cache.

Also:
- Update to ORAS 0.6.0
- Simplify registry client setup with NewClientWithDefaults()
- Remove needless annotations and constants

Fixes #6068
Fixes #6141

Signed-off-by: Josh Dolitsky <jdolitsky@gmail.com>
pull/6207/head
Josh Dolitsky 6 years ago committed by GitHub
parent 6ab25d2242
commit 6095070817
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

486
Gopkg.lock generated

File diff suppressed because it is too large Load Diff

@ -44,7 +44,7 @@
[[constraint]]
name = "github.com/deislabs/oras"
version = "0.5.0"
version = "0.6.0"
[[constraint]]
name = "github.com/sirupsen/logrus"

@ -17,17 +17,13 @@ limitations under the License.
package main // import "helm.sh/helm/cmd/helm"
import (
"context"
"io"
"path/filepath"
auth "github.com/deislabs/oras/pkg/auth/docker"
"github.com/spf13/cobra"
"helm.sh/helm/cmd/helm/require"
"helm.sh/helm/internal/experimental/registry"
"helm.sh/helm/pkg/action"
"helm.sh/helm/pkg/helmpath"
)
const (
@ -132,30 +128,7 @@ func newRootCmd(actionConfig *action.Configuration, out io.Writer, args []string
// set defaults from environment
settings.Init(flags)
// Add the registry client based on settings
// TODO: Move this elsewhere (first, settings.Init() must move)
// TODO: handle errors, dont panic
credentialsFile := filepath.Join(helmpath.Registry(), registry.CredentialsFileBasename)
client, err := auth.NewClient(credentialsFile)
if err != nil {
panic(err)
}
resolver, err := client.Resolver(context.Background())
if err != nil {
panic(err)
}
actionConfig.RegistryClient = registry.NewClient(&registry.ClientOptions{
Debug: settings.Debug,
Out: out,
Authorizer: registry.Authorizer{
Client: client,
},
Resolver: registry.Resolver{
Resolver: resolver,
},
CacheRootDir: helmpath.Registry(),
})
// Add subcommands
cmd.AddCommand(
// chart commands
newCreateCmd(out),
@ -168,10 +141,6 @@ func newRootCmd(actionConfig *action.Configuration, out io.Writer, args []string
newSearchCmd(out),
newVerifyCmd(out),
// registry/chart cache commands
newRegistryCmd(actionConfig, out),
newChartCmd(actionConfig, out),
// release commands
newGetCmd(actionConfig, out),
newHistoryCmd(actionConfig, out),
@ -193,6 +162,21 @@ func newRootCmd(actionConfig *action.Configuration, out io.Writer, args []string
newDocsCmd(out),
)
// Add *experimental* subcommands
registryClient, err := registry.NewClient(
registry.ClientOptDebug(settings.Debug),
registry.ClientOptWriter(out),
)
if err != nil {
// TODO: dont panic here, refactor newRootCmd to return error
panic(err)
}
actionConfig.RegistryClient = registryClient
cmd.AddCommand(
newRegistryCmd(actionConfig, out),
newChartCmd(actionConfig, out),
)
// Find and add plugins
loadPlugins(cmd, out)

@ -24,13 +24,13 @@ import (
"io/ioutil"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
orascontent "github.com/deislabs/oras/pkg/content"
units "github.com/docker/go-units"
checksum "github.com/opencontainers/go-digest"
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
@ -39,451 +39,324 @@ import (
"helm.sh/helm/pkg/chartutil"
)
var (
tableHeaders = []string{"name", "version", "digest", "size", "created"}
const (
// CacheRootDir is the root directory for a cache
CacheRootDir = "cache"
)
type (
filesystemCache struct {
out io.Writer
rootDir string
store *orascontent.Memorystore
// Cache handles local/in-memory storage of Helm charts, compliant with OCI Layout
Cache struct {
debug bool
out io.Writer
rootDir string
ociStore *orascontent.OCIStore
memoryStore *orascontent.Memorystore
}
// CacheRefSummary contains as much info as available describing a chart reference in cache
// Note: fields here are sorted by the order in which they are set in FetchReference method
CacheRefSummary struct {
Name string
Repo string
Tag string
Exists bool
Manifest *ocispec.Descriptor
Config *ocispec.Descriptor
ContentLayer *ocispec.Descriptor
Size int64
Digest digest.Digest
CreatedAt time.Time
Chart *chart.Chart
}
)
func (cache *filesystemCache) LayersToChart(layers []ocispec.Descriptor) (*chart.Chart, error) {
metaLayer, contentLayer, err := extractLayers(layers)
if err != nil {
return nil, err
// NewCache returns a new OCI Layout-compliant cache with config
func NewCache(opts ...CacheOption) (*Cache, error) {
cache := &Cache{
out: ioutil.Discard,
}
name, version, err := extractChartNameVersionFromLayer(contentLayer)
if err != nil {
return nil, err
for _, opt := range opts {
opt(cache)
}
// Obtain raw chart meta content (json)
_, metaJSONRaw, ok := cache.store.Get(metaLayer)
if !ok {
return nil, errors.New("error retrieving meta layer")
// validate
if cache.rootDir == "" {
return nil, errors.New("must set cache root dir on initialization")
}
return cache, nil
}
// Construct chart metadata object
metadata := chart.Metadata{}
err = json.Unmarshal(metaJSONRaw, &metadata)
if err != nil {
// FetchReference retrieves a chart ref from cache
func (cache *Cache) FetchReference(ref *Reference) (*CacheRefSummary, error) {
if err := cache.init(); err != nil {
return nil, err
}
metadata.APIVersion = chart.APIVersionV1
metadata.Name = name
metadata.Version = version
// Obtain raw chart content
_, contentRaw, ok := cache.store.Get(contentLayer)
if !ok {
return nil, errors.New("error retrieving meta layer")
}
// Construct chart object and attach metadata
ch, err := loader.LoadArchive(bytes.NewBuffer(contentRaw))
if err != nil {
return nil, err
r := CacheRefSummary{
Name: ref.FullName(),
Repo: ref.Repo,
Tag: ref.Tag,
}
for _, desc := range cache.ociStore.ListReferences() {
if desc.Annotations[ocispec.AnnotationRefName] == r.Name {
r.Exists = true
manifestBytes, err := cache.fetchBlob(&desc)
if err != nil {
return &r, err
}
var manifest ocispec.Manifest
err = json.Unmarshal(manifestBytes, &manifest)
if err != nil {
return &r, err
}
r.Manifest = &desc
r.Config = &manifest.Config
numLayers := len(manifest.Layers)
if numLayers != 1 {
return &r, errors.New(
fmt.Sprintf("manifest does not contain exactly 1 layer (total: %d)", numLayers))
}
var contentLayer *ocispec.Descriptor
for _, layer := range manifest.Layers {
switch layer.MediaType {
case HelmChartContentLayerMediaType:
contentLayer = &layer
}
}
if contentLayer.Size == 0 {
return &r, errors.New(
fmt.Sprintf("manifest does not contain a layer with mediatype %s", HelmChartContentLayerMediaType))
}
r.ContentLayer = contentLayer
info, err := cache.ociStore.Info(ctx(cache.out, cache.debug), contentLayer.Digest)
if err != nil {
return &r, err
}
r.Size = info.Size
r.Digest = info.Digest
r.CreatedAt = info.CreatedAt
contentBytes, err := cache.fetchBlob(contentLayer)
if err != nil {
return &r, err
}
ch, err := loader.LoadArchive(bytes.NewBuffer(contentBytes))
if err != nil {
return &r, err
}
r.Chart = ch
}
}
ch.Metadata = &metadata
return ch, nil
return &r, nil
}
func (cache *filesystemCache) ChartToLayers(ch *chart.Chart) ([]ocispec.Descriptor, error) {
// extract/separate the name and version from other metadata
if err := ch.Validate(); err != nil {
// StoreReference stores a chart ref in cache
func (cache *Cache) StoreReference(ref *Reference, ch *chart.Chart) (*CacheRefSummary, error) {
if err := cache.init(); err != nil {
return nil, err
}
name := ch.Metadata.Name
version := ch.Metadata.Version
// Create meta layer, clear name and version from Chart.yaml and convert to json
ch.Metadata.Name = ""
ch.Metadata.Version = ""
metaJSONRaw, err := json.Marshal(ch.Metadata)
if err != nil {
return nil, err
}
metaLayer := cache.store.Add(HelmChartMetaFileName, HelmChartMetaLayerMediaType, metaJSONRaw)
// Create content layer
// TODO: something better than this hack. Currently needed for chartutil.Save()
// If metadata does not contain Name or Version, an error is returned
// such as "no chart name specified (Chart.yaml)"
ch.Metadata = &chart.Metadata{
APIVersion: chart.APIVersionV1,
Name: "-",
Version: "0.1.0",
}
destDir := mkdir(filepath.Join(cache.rootDir, "blobs", ".build"))
tmpFile, err := chartutil.Save(ch, destDir)
defer os.Remove(tmpFile)
if err != nil {
return nil, errors.Wrap(err, "failed to save")
r := CacheRefSummary{
Name: ref.FullName(),
Repo: ref.Repo,
Tag: ref.Tag,
Chart: ch,
}
contentRaw, err := ioutil.ReadFile(tmpFile)
existing, _ := cache.FetchReference(ref)
r.Exists = existing.Exists
config, _, err := cache.saveChartConfig(ch)
if err != nil {
return nil, err
return &r, err
}
contentLayer := cache.store.Add(HelmChartContentFileName, HelmChartContentLayerMediaType, contentRaw)
// Set annotations
contentLayer.Annotations[HelmChartNameAnnotation] = name
contentLayer.Annotations[HelmChartVersionAnnotation] = version
layers := []ocispec.Descriptor{metaLayer, contentLayer}
return layers, nil
}
func (cache *filesystemCache) LoadReference(ref *Reference) ([]ocispec.Descriptor, error) {
tagDir := filepath.Join(cache.rootDir, "refs", escape(ref.Repo), "tags", ref.Tag)
// add meta layer
metaJSONRaw, err := getSymlinkDestContent(filepath.Join(tagDir, "meta"))
r.Config = config
contentLayer, _, err := cache.saveChartContentLayer(ch)
if err != nil {
return nil, err
return &r, err
}
metaLayer := cache.store.Add(HelmChartMetaFileName, HelmChartMetaLayerMediaType, metaJSONRaw)
// add content layer
contentRaw, err := getSymlinkDestContent(filepath.Join(tagDir, "content"))
r.ContentLayer = contentLayer
info, err := cache.ociStore.Info(ctx(cache.out, cache.debug), contentLayer.Digest)
if err != nil {
return nil, err
return &r, err
}
contentLayer := cache.store.Add(HelmChartContentFileName, HelmChartContentLayerMediaType, contentRaw)
// set annotations on content layer (chart name and version)
err = setLayerAnnotationsFromChartLink(contentLayer, filepath.Join(tagDir, "chart"))
r.Size = info.Size
r.Digest = info.Digest
r.CreatedAt = info.CreatedAt
manifest, _, err := cache.saveChartManifest(config, contentLayer)
if err != nil {
return nil, err
return &r, err
}
printChartSummary(cache.out, metaLayer, contentLayer)
layers := []ocispec.Descriptor{metaLayer, contentLayer}
return layers, nil
r.Manifest = manifest
return &r, nil
}
func (cache *filesystemCache) StoreReference(ref *Reference, layers []ocispec.Descriptor) (bool, error) {
tagDir := mkdir(filepath.Join(cache.rootDir, "refs", escape(ref.Repo), "tags", ref.Tag))
// Retrieve just the meta and content layers
metaLayer, contentLayer, err := extractLayers(layers)
if err != nil {
return false, err
}
// Extract chart name and version
name, version, err := extractChartNameVersionFromLayer(contentLayer)
if err != nil {
return false, err
// DeleteReference deletes a chart ref from cache
// TODO: garbage collection, only manifest removed
func (cache *Cache) DeleteReference(ref *Reference) (*CacheRefSummary, error) {
if err := cache.init(); err != nil {
return nil, err
}
// Create chart file
chartPath, err := createChartFile(filepath.Join(cache.rootDir, "charts"), name, version)
if err != nil {
return false, err
r, err := cache.FetchReference(ref)
if err != nil || !r.Exists {
return r, err
}
cache.ociStore.DeleteReference(r.Name)
err = cache.ociStore.SaveIndex()
return r, err
}
// Create chart symlink
err = createSymlink(chartPath, filepath.Join(tagDir, "chart"))
if err != nil {
return false, err
// ListReferences lists all chart refs in a cache
func (cache *Cache) ListReferences() ([]*CacheRefSummary, error) {
if err := cache.init(); err != nil {
return nil, err
}
// Save meta blob
metaExists, metaPath := digestPath(filepath.Join(cache.rootDir, "blobs"), metaLayer.Digest)
if !metaExists {
fmt.Fprintf(cache.out, "%s: Saving meta (%s)\n",
shortDigest(metaLayer.Digest.Hex()), byteCountBinary(metaLayer.Size))
_, metaJSONRaw, ok := cache.store.Get(metaLayer)
if !ok {
return false, errors.New("error retrieving meta layer")
var rr []*CacheRefSummary
for _, desc := range cache.ociStore.ListReferences() {
name := desc.Annotations[ocispec.AnnotationRefName]
if name == "" {
if cache.debug {
fmt.Fprintf(cache.out, "warning: found manifest without name: %s", desc.Digest.Hex())
}
continue
}
err = writeFile(metaPath, metaJSONRaw)
ref, err := ParseReference(name)
if err != nil {
return false, err
return rr, err
}
}
// Create meta symlink
err = createSymlink(metaPath, filepath.Join(tagDir, "meta"))
if err != nil {
return false, err
}
// Save content blob
contentExists, contentPath := digestPath(filepath.Join(cache.rootDir, "blobs"), contentLayer.Digest)
if !contentExists {
fmt.Fprintf(cache.out, "%s: Saving content (%s)\n",
shortDigest(contentLayer.Digest.Hex()), byteCountBinary(contentLayer.Size))
_, contentRaw, ok := cache.store.Get(contentLayer)
if !ok {
return false, errors.New("error retrieving content layer")
}
err = writeFile(contentPath, contentRaw)
r, err := cache.FetchReference(ref)
if err != nil {
return false, err
return rr, err
}
rr = append(rr, r)
}
// Create content symlink
err = createSymlink(contentPath, filepath.Join(tagDir, "content"))
if err != nil {
return false, err
}
printChartSummary(cache.out, metaLayer, contentLayer)
return metaExists && contentExists, nil
return rr, nil
}
func (cache *filesystemCache) DeleteReference(ref *Reference) error {
tagDir := filepath.Join(cache.rootDir, "refs", escape(ref.Repo), "tags", ref.Tag)
if _, err := os.Stat(tagDir); os.IsNotExist(err) {
return errors.New("ref not found")
// AddManifest provides a manifest to the cache index.json
func (cache *Cache) AddManifest(ref *Reference, manifest *ocispec.Descriptor) error {
if err := cache.init(); err != nil {
return err
}
return os.RemoveAll(tagDir)
}
func (cache *filesystemCache) TableRows() ([][]interface{}, error) {
return getRefsSorted(filepath.Join(cache.rootDir, "refs"))
cache.ociStore.AddReference(ref.FullName(), *manifest)
err := cache.ociStore.SaveIndex()
return err
}
// escape sanitizes a registry URL to remove characters such as ":"
// which are illegal on windows
func escape(s string) string {
return strings.ReplaceAll(s, ":", "_")
// Provider provides a valid containerd Provider
func (cache *Cache) Provider() content.Provider {
return content.Provider(cache.ociStore)
}
// escape reverses escape
func unescape(s string) string {
return strings.ReplaceAll(s, "_", ":")
// Ingester provides a valid containerd Ingester
func (cache *Cache) Ingester() content.Ingester {
return content.Ingester(cache.ociStore)
}
// printChartSummary prints details about a chart layers
func printChartSummary(out io.Writer, metaLayer ocispec.Descriptor, contentLayer ocispec.Descriptor) {
fmt.Fprintf(out, "Name: %s\n", contentLayer.Annotations[HelmChartNameAnnotation])
fmt.Fprintf(out, "Version: %s\n", contentLayer.Annotations[HelmChartVersionAnnotation])
fmt.Fprintf(out, "Meta: %s\n", metaLayer.Digest)
fmt.Fprintf(out, "Content: %s\n", contentLayer.Digest)
// ProvideIngester provides a valid oras ProvideIngester
func (cache *Cache) ProvideIngester() orascontent.ProvideIngester {
return orascontent.ProvideIngester(cache.ociStore)
}
// fileExists determines if a file exists
func fileExists(path string) bool {
if _, err := os.Stat(path); os.IsNotExist(err) {
return false
// init creates files needed necessary for OCI layout store
func (cache *Cache) init() error {
if cache.ociStore == nil {
ociStore, err := orascontent.NewOCIStore(cache.rootDir)
if err != nil {
return err
}
cache.ociStore = ociStore
cache.memoryStore = orascontent.NewMemoryStore()
}
return true
}
// mkdir will create a directory (no error check) and return the path
func mkdir(dir string) string {
os.MkdirAll(dir, 0755)
return dir
}
// createSymlink creates a symbolic link, deleting existing one if exists
func createSymlink(src string, dest string) error {
os.Remove(dest)
err := os.Symlink(src, dest)
return err
return nil
}
// getSymlinkDestContent returns the file contents of a symlink's destination
func getSymlinkDestContent(linkPath string) ([]byte, error) {
src, err := os.Readlink(linkPath)
// saveChartConfig stores the Chart.yaml as json blob and returns a descriptor
func (cache *Cache) saveChartConfig(ch *chart.Chart) (*ocispec.Descriptor, bool, error) {
configBytes, err := json.Marshal(ch.Metadata)
if err != nil {
return nil, err
return nil, false, err
}
return ioutil.ReadFile(src)
configExists, err := cache.storeBlob(configBytes)
if err != nil {
return nil, configExists, err
}
descriptor := cache.memoryStore.Add("", HelmChartConfigMediaType, configBytes)
return &descriptor, configExists, nil
}
// setLayerAnnotationsFromChartLink will set chart name/version annotations on a layer
// based on the path of the chart link destination
func setLayerAnnotationsFromChartLink(layer ocispec.Descriptor, chartLinkPath string) error {
src, err := os.Readlink(chartLinkPath)
// saveChartContentLayer stores the chart as tarball blob and returns a descriptor
func (cache *Cache) saveChartContentLayer(ch *chart.Chart) (*ocispec.Descriptor, bool, error) {
destDir := filepath.Join(cache.rootDir, ".build")
os.MkdirAll(destDir, 0755)
tmpFile, err := chartutil.Save(ch, destDir)
defer os.Remove(tmpFile)
if err != nil {
return err
return nil, false, errors.Wrap(err, "failed to save")
}
// example path: /some/path/charts/mychart/versions/1.2.0
chartName := filepath.Base(filepath.Dir(filepath.Dir(src)))
chartVersion := filepath.Base(src)
layer.Annotations[HelmChartNameAnnotation] = chartName
layer.Annotations[HelmChartVersionAnnotation] = chartVersion
return nil
contentBytes, err := ioutil.ReadFile(tmpFile)
if err != nil {
return nil, false, err
}
contentExists, err := cache.storeBlob(contentBytes)
if err != nil {
return nil, contentExists, err
}
descriptor := cache.memoryStore.Add("", HelmChartContentLayerMediaType, contentBytes)
return &descriptor, contentExists, nil
}
// extractLayers obtains the meta and content layers from a list of layers
func extractLayers(layers []ocispec.Descriptor) (ocispec.Descriptor, ocispec.Descriptor, error) {
var metaLayer, contentLayer ocispec.Descriptor
if len(layers) != 2 {
return metaLayer, contentLayer, errors.New("manifest does not contain exactly 2 layers")
// saveChartManifest stores the chart manifest as json blob and returns a descriptor
func (cache *Cache) saveChartManifest(config *ocispec.Descriptor, contentLayer *ocispec.Descriptor) (*ocispec.Descriptor, bool, error) {
manifest := ocispec.Manifest{
Versioned: specs.Versioned{SchemaVersion: 2},
Config: *config,
Layers: []ocispec.Descriptor{*contentLayer},
}
for _, layer := range layers {
switch layer.MediaType {
case HelmChartMetaLayerMediaType:
metaLayer = layer
case HelmChartContentLayerMediaType:
contentLayer = layer
}
manifestBytes, err := json.Marshal(manifest)
if err != nil {
return nil, false, err
}
if metaLayer.Size == 0 {
return metaLayer, contentLayer, errors.New("manifest does not contain a Helm chart meta layer")
manifestExists, err := cache.storeBlob(manifestBytes)
if err != nil {
return nil, manifestExists, err
}
if contentLayer.Size == 0 {
return metaLayer, contentLayer, errors.New("manifest does not contain a Helm chart content layer")
descriptor := ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageManifest,
Digest: digest.FromBytes(manifestBytes),
Size: int64(len(manifestBytes)),
}
return metaLayer, contentLayer, nil
return &descriptor, manifestExists, nil
}
// extractChartNameVersionFromLayer retrieves the chart name and version from layer annotations
func extractChartNameVersionFromLayer(layer ocispec.Descriptor) (string, string, error) {
name, ok := layer.Annotations[HelmChartNameAnnotation]
if !ok {
return "", "", errors.New("could not find chart name in annotations")
// storeBlob stores a blob on filesystem
func (cache *Cache) storeBlob(blobBytes []byte) (bool, error) {
var exists bool
writer, err := cache.ociStore.Store.Writer(ctx(cache.out, cache.debug),
content.WithRef(digest.FromBytes(blobBytes).Hex()))
if err != nil {
return exists, err
}
version, ok := layer.Annotations[HelmChartVersionAnnotation]
if !ok {
return "", "", errors.New("could not find chart version in annotations")
_, err = writer.Write(blobBytes)
if err != nil {
return exists, err
}
return name, version, nil
}
// createChartFile creates a file under "<chartsdir>" dir which is linked to by ref
func createChartFile(chartsRootDir string, name string, version string) (string, error) {
chartPathDir := filepath.Join(chartsRootDir, name, "versions")
chartPath := filepath.Join(chartPathDir, version)
if _, err := os.Stat(chartPath); err != nil && os.IsNotExist(err) {
os.MkdirAll(chartPathDir, 0755)
err := ioutil.WriteFile(chartPath, []byte("-"), 0644)
if err != nil {
return "", err
err = writer.Commit(ctx(cache.out, cache.debug), 0, writer.Digest())
if err != nil {
if !errdefs.IsAlreadyExists(err) {
return exists, err
}
exists = true
}
return chartPath, nil
}
// digestPath returns the path to addressable content, and whether the file exists
func digestPath(rootDir string, digest checksum.Digest) (bool, string) {
path := filepath.Join(rootDir, "sha256", digest.Hex())
exists := fileExists(path)
return exists, path
}
// writeFile creates a path, ensuring parent directory
func writeFile(path string, c []byte) error {
os.MkdirAll(filepath.Dir(path), 0755)
return ioutil.WriteFile(path, c, 0644)
}
// byteCountBinary produces a human-readable file size
func byteCountBinary(b int64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
err = writer.Close()
return exists, err
}
// shortDigest returns first 7 characters of a sha256 digest
func shortDigest(digest string) string {
if len(digest) == 64 {
return digest[:7]
}
return digest
}
// getRefsSorted returns a map of all refs stored in a refsRootDir
func getRefsSorted(refsRootDir string) ([][]interface{}, error) {
refsMap := map[string]map[string]string{}
// Walk the storage dir, check for symlinks under "refs" dir pointing to valid files in "blobs/" and "charts/"
err := filepath.Walk(refsRootDir, func(path string, fileInfo os.FileInfo, fileError error) error {
// Check if this file is a symlink
linkPath, err := os.Readlink(path)
if err == nil {
destFileInfo, err := os.Stat(linkPath)
if err == nil {
tagDir := filepath.Dir(path)
// Determine the ref
repo := unescape(strings.TrimLeft(
strings.TrimPrefix(filepath.Dir(filepath.Dir(tagDir)), refsRootDir), "/\\"))
tag := filepath.Base(tagDir)
ref := fmt.Sprintf("%s:%s", repo, tag)
// Init hashmap entry if does not exist
if _, ok := refsMap[ref]; !ok {
refsMap[ref] = map[string]string{}
}
// Add data to entry based on file name (symlink name)
base := filepath.Base(path)
switch base {
case "chart":
refsMap[ref]["name"] = filepath.Base(filepath.Dir(filepath.Dir(linkPath)))
refsMap[ref]["version"] = destFileInfo.Name()
case "content":
// Make sure the filename looks like a sha256 digest (64 chars)
digest := destFileInfo.Name()
if len(digest) == 64 {
refsMap[ref]["digest"] = shortDigest(digest)
refsMap[ref]["size"] = byteCountBinary(destFileInfo.Size())
refsMap[ref]["created"] = units.HumanDuration(time.Now().UTC().Sub(destFileInfo.ModTime()))
}
}
}
}
return nil
})
// Filter out any refs that are incomplete (do not have all required fields)
for k, ref := range refsMap {
allKeysFound := true
for _, v := range tableHeaders {
if _, ok := ref[v]; !ok {
allKeysFound = false
break
}
}
if !allKeysFound {
delete(refsMap, k)
}
// fetchBlob retrieves a blob from filesystem
func (cache *Cache) fetchBlob(desc *ocispec.Descriptor) ([]byte, error) {
reader, err := cache.ociStore.ReaderAt(ctx(cache.out, cache.debug), *desc)
if err != nil {
return nil, err
}
// Sort and convert to format expected by uitable
refs := make([][]interface{}, len(refsMap))
keys := make([]string, 0, len(refsMap))
for key := range refsMap {
keys = append(keys, key)
}
sort.Strings(keys)
for i, key := range keys {
refs[i] = make([]interface{}, len(tableHeaders)+1)
refs[i][0] = key
ref := refsMap[key]
for j, k := range tableHeaders {
refs[i][j+1] = ref[k]
}
bytes := make([]byte, desc.Size)
_, err = reader.ReadAt(bytes, 0)
if err != nil {
return nil, err
}
return refs, err
return bytes, nil
}

@ -0,0 +1,48 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry // import "helm.sh/helm/internal/experimental/registry"
import (
"io"
)
type (
// CacheOption allows specifying various settings configurable by the user for overriding the defaults
// used when creating a new default cache
CacheOption func(*Cache)
)
// CacheOptDebug returns a function that sets the debug setting on cache options set
func CacheOptDebug(debug bool) CacheOption {
return func(cache *Cache) {
cache.debug = debug
}
}
// CacheOptWriter returns a function that sets the writer setting on cache options set
func CacheOptWriter(out io.Writer) CacheOption {
return func(cache *Cache) {
cache.out = out
}
}
// CacheOptRoot returns a function that sets the root directory setting on cache options set
func CacheOptRoot(rootDir string) CacheOption {
return func(cache *Cache) {
cache.rootDir = rootDir
}
}

@ -20,147 +20,201 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"path/filepath"
"sort"
orascontent "github.com/deislabs/oras/pkg/content"
orascontext "github.com/deislabs/oras/pkg/context"
auth "github.com/deislabs/oras/pkg/auth/docker"
"github.com/deislabs/oras/pkg/oras"
"github.com/gosuri/uitable"
"github.com/sirupsen/logrus"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"helm.sh/helm/pkg/chart"
"helm.sh/helm/pkg/helmpath"
)
const (
// CredentialsFileBasename is the filename for auth credentials file
CredentialsFileBasename = "config.json"
)
type (
// ClientOptions is used to construct a new client
ClientOptions struct {
Debug bool
Out io.Writer
Authorizer Authorizer
Resolver Resolver
CacheRootDir string
}
// Client works with OCI-compliant registries and local Helm chart cache
Client struct {
debug bool
out io.Writer
authorizer Authorizer
resolver Resolver
cache *filesystemCache // TODO: something more robust
authorizer *Authorizer
resolver *Resolver
cache *Cache
}
)
// NewClient returns a new registry client with config
func NewClient(options *ClientOptions) *Client {
return &Client{
debug: options.Debug,
out: options.Out,
resolver: options.Resolver,
authorizer: options.Authorizer,
cache: &filesystemCache{
out: options.Out,
rootDir: options.CacheRootDir,
store: orascontent.NewMemoryStore(),
},
func NewClient(opts ...ClientOption) (*Client, error) {
client := &Client{
out: ioutil.Discard,
}
for _, opt := range opts {
opt(client)
}
// set defaults if fields are missing
if client.authorizer == nil {
credentialsFile := filepath.Join(helmpath.Registry(), CredentialsFileBasename)
authClient, err := auth.NewClient(credentialsFile)
if err != nil {
return nil, err
}
client.authorizer = &Authorizer{
Client: authClient,
}
}
if client.resolver == nil {
resolver, err := client.authorizer.Resolver(context.Background())
if err != nil {
return nil, err
}
client.resolver = &Resolver{
Resolver: resolver,
}
}
if client.cache == nil {
cache, err := NewCache(
CacheOptDebug(client.debug),
CacheOptWriter(client.out),
CacheOptRoot(filepath.Join(helmpath.Registry(), CacheRootDir)),
)
if err != nil {
return nil, err
}
client.cache = cache
}
return client, nil
}
// Login logs into a registry
func (c *Client) Login(hostname string, username string, password string) error {
err := c.authorizer.Login(c.newContext(), hostname, username, password)
err := c.authorizer.Login(ctx(c.out, c.debug), hostname, username, password)
if err != nil {
return err
}
fmt.Fprint(c.out, "Login succeeded\n")
fmt.Fprintf(c.out, "Login succeeded\n")
return nil
}
// Logout logs out of a registry
func (c *Client) Logout(hostname string) error {
err := c.authorizer.Logout(c.newContext(), hostname)
err := c.authorizer.Logout(ctx(c.out, c.debug), hostname)
if err != nil {
return err
}
fmt.Fprint(c.out, "Logout succeeded\n")
fmt.Fprintln(c.out, "Logout succeeded")
return nil
}
// PushChart uploads a chart to a registry
func (c *Client) PushChart(ref *Reference) error {
fmt.Fprintf(c.out, "The push refers to repository [%s]\n", ref.Repo)
layers, err := c.cache.LoadReference(ref)
r, err := c.cache.FetchReference(ref)
if err != nil {
return err
}
_, err = oras.Push(c.newContext(), c.resolver, ref.String(), c.cache.store, layers,
oras.WithConfigMediaType(HelmChartConfigMediaType))
if !r.Exists {
return errors.New(fmt.Sprintf("Chart not found: %s", r.Name))
}
fmt.Fprintf(c.out, "The push refers to repository [%s]\n", r.Repo)
c.printCacheRefSummary(r)
layers := []ocispec.Descriptor{*r.ContentLayer}
_, err = oras.Push(ctx(c.out, c.debug), c.resolver, r.Name, c.cache.Provider(), layers,
oras.WithConfig(*r.Config), oras.WithNameValidation(nil))
if err != nil {
return err
}
var totalSize int64
for _, layer := range layers {
totalSize += layer.Size
s := ""
numLayers := len(layers)
if 1 < numLayers {
s = "s"
}
fmt.Fprintf(c.out,
"%s: pushed to remote (%d layers, %s total)\n", ref.Tag, len(layers), byteCountBinary(totalSize))
"%s: pushed to remote (%d layer%s, %s total)\n", r.Tag, numLayers, s, byteCountBinary(r.Size))
return nil
}
// PullChart downloads a chart from a registry
func (c *Client) PullChart(ref *Reference) error {
if ref.Tag == "" {
return errors.New("tag explicitly required")
}
existing, err := c.cache.FetchReference(ref)
if err != nil {
return err
}
fmt.Fprintf(c.out, "%s: Pulling from %s\n", ref.Tag, ref.Repo)
_, layers, err := oras.Pull(c.newContext(), c.resolver, ref.String(), c.cache.store, oras.WithAllowedMediaTypes(KnownMediaTypes()))
manifest, _, err := oras.Pull(ctx(c.out, c.debug), c.resolver, ref.FullName(), c.cache.Ingester(),
oras.WithPullEmptyNameAllowed(),
oras.WithAllowedMediaTypes(KnownMediaTypes()),
oras.WithContentProvideIngester(c.cache.ProvideIngester()))
if err != nil {
return err
}
err = c.cache.AddManifest(ref, &manifest)
if err != nil {
return err
}
exists, err := c.cache.StoreReference(ref, layers)
r, err := c.cache.FetchReference(ref)
if err != nil {
return err
}
if !exists {
fmt.Fprintf(c.out, "Status: Downloaded newer chart for %s:%s\n", ref.Repo, ref.Tag)
if !r.Exists {
return errors.New(fmt.Sprintf("Chart not found: %s", r.Name))
}
c.printCacheRefSummary(r)
if !existing.Exists {
fmt.Fprintf(c.out, "Status: Downloaded newer chart for %s\n", ref.FullName())
} else {
fmt.Fprintf(c.out, "Status: Chart is up to date for %s:%s\n", ref.Repo, ref.Tag)
fmt.Fprintf(c.out, "Status: Chart is up to date for %s\n", ref.FullName())
}
return nil
return err
}
// SaveChart stores a copy of chart in local cache
func (c *Client) SaveChart(ch *chart.Chart, ref *Reference) error {
layers, err := c.cache.ChartToLayers(ch)
r, err := c.cache.StoreReference(ref, ch)
if err != nil {
return err
}
_, err = c.cache.StoreReference(ref, layers)
c.printCacheRefSummary(r)
err = c.cache.AddManifest(ref, r.Manifest)
if err != nil {
return err
}
fmt.Fprintf(c.out, "%s: saved\n", ref.Tag)
fmt.Fprintf(c.out, "%s: saved\n", r.Tag)
return nil
}
// LoadChart retrieves a chart object by reference
func (c *Client) LoadChart(ref *Reference) (*chart.Chart, error) {
layers, err := c.cache.LoadReference(ref)
r, err := c.cache.FetchReference(ref)
if err != nil {
return nil, err
}
ch, err := c.cache.LayersToChart(layers)
return ch, err
if !r.Exists {
return nil, errors.New(fmt.Sprintf("Chart not found: %s", ref.FullName()))
}
c.printCacheRefSummary(r)
return r.Chart, nil
}
// RemoveChart deletes a locally saved chart
func (c *Client) RemoveChart(ref *Reference) error {
err := c.cache.DeleteReference(ref)
r, err := c.cache.DeleteReference(ref)
if err != nil {
return err
}
fmt.Fprintf(c.out, "%s: removed\n", ref.Tag)
return err
if !r.Exists {
return errors.New(fmt.Sprintf("Chart not found: %s", ref.FullName()))
}
fmt.Fprintf(c.out, "%s: removed\n", r.Tag)
return nil
}
// PrintChartTable prints a list of locally stored charts
@ -168,7 +222,7 @@ func (c *Client) PrintChartTable() error {
table := uitable.New()
table.MaxColWidth = 60
table.AddRow("REF", "NAME", "VERSION", "DIGEST", "SIZE", "CREATED")
rows, err := c.cache.TableRows()
rows, err := c.getChartTableRows()
if err != nil {
return err
}
@ -179,12 +233,45 @@ func (c *Client) PrintChartTable() error {
return nil
}
// disable verbose logging coming from ORAS unless debug is enabled
func (c *Client) newContext() context.Context {
if !c.debug {
return orascontext.Background()
// printCacheRefSummary prints out chart ref summary
func (c *Client) printCacheRefSummary(r *CacheRefSummary) {
fmt.Fprintf(c.out, "ref: %s\n", r.Name)
fmt.Fprintf(c.out, "digest: %s\n", r.Digest.Hex())
fmt.Fprintf(c.out, "size: %s\n", byteCountBinary(r.Size))
fmt.Fprintf(c.out, "name: %s\n", r.Chart.Metadata.Name)
fmt.Fprintf(c.out, "version: %s\n", r.Chart.Metadata.Version)
}
// getChartTableRows returns rows in uitable-friendly format
func (c *Client) getChartTableRows() ([][]interface{}, error) {
rr, err := c.cache.ListReferences()
if err != nil {
return nil, err
}
refsMap := map[string]map[string]string{}
for _, r := range rr {
refsMap[r.Name] = map[string]string{
"name": r.Chart.Metadata.Name,
"version": r.Chart.Metadata.Version,
"digest": shortDigest(r.Digest.Hex()),
"size": byteCountBinary(r.Size),
"created": timeAgo(r.CreatedAt),
}
}
// Sort and convert to format expected by uitable
rows := make([][]interface{}, len(refsMap))
keys := make([]string, 0, len(refsMap))
for key := range refsMap {
keys = append(keys, key)
}
sort.Strings(keys)
for i, key := range keys {
rows[i] = make([]interface{}, 6)
rows[i][0] = key
ref := refsMap[key]
for j, k := range []string{"name", "version", "digest", "size", "created"} {
rows[i][j+1] = ref[k]
}
}
ctx := orascontext.WithLoggerFromWriter(context.Background(), c.out)
orascontext.GetLogger(ctx).Logger.SetLevel(logrus.DebugLevel)
return ctx
return rows, nil
}

@ -0,0 +1,62 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry // import "helm.sh/helm/internal/experimental/registry"
import (
"io"
)
type (
// ClientOption allows specifying various settings configurable by the user for overriding the defaults
// used when creating a new default client
ClientOption func(*Client)
)
// ClientOptDebug returns a function that sets the debug setting on client options set
func ClientOptDebug(debug bool) ClientOption {
return func(client *Client) {
client.debug = debug
}
}
// ClientOptWriter returns a function that sets the writer setting on client options set
func ClientOptWriter(out io.Writer) ClientOption {
return func(client *Client) {
client.out = out
}
}
// ClientOptResolver returns a function that sets the resolver setting on client options set
func ClientOptResolver(resolver *Resolver) ClientOption {
return func(client *Client) {
client.resolver = resolver
}
}
// ClientOptAuthorizer returns a function that sets the authorizer setting on client options set
func ClientOptAuthorizer(authorizer *Authorizer) ClientOption {
return func(client *Client) {
client.authorizer = authorizer
}
}
// ClientOptCache returns a function that sets the cache setting on a client options set
func ClientOptCache(cache *Cache) ClientOption {
return func(client *Client) {
client.cache = cache
}
}

@ -69,17 +69,27 @@ func (suite *RegistryClientTestSuite) SetupSuite() {
resolver, err := client.Resolver(context.Background())
suite.Nil(err, "no error creating resolver")
// Init test client
suite.RegistryClient = NewClient(&ClientOptions{
Out: suite.Out,
Authorizer: Authorizer{
// create cache
cache, err := NewCache(
CacheOptDebug(true),
CacheOptWriter(suite.Out),
CacheOptRoot(filepath.Join(suite.CacheRootDir, CacheRootDir)),
)
suite.Nil(err, "no error creating cache")
// init test client
suite.RegistryClient, err = NewClient(
ClientOptDebug(true),
ClientOptWriter(suite.Out),
ClientOptAuthorizer(&Authorizer{
Client: client,
},
Resolver: Resolver{
}),
ClientOptResolver(&Resolver{
Resolver: resolver,
},
CacheRootDir: suite.CacheRootDir,
})
}),
ClientOptCache(cache),
)
suite.Nil(err, "no error creating registry client")
// create htpasswd file (w BCrypt, which is required)
pwBytes, err := bcrypt.GenerateFromPassword([]byte(testPassword), bcrypt.DefaultCost)

@ -20,29 +20,14 @@ const (
// HelmChartConfigMediaType is the reserved media type for the Helm chart manifest config
HelmChartConfigMediaType = "application/vnd.cncf.helm.config.v1+json"
// HelmChartMetaLayerMediaType is the reserved media type for Helm chart metadata
HelmChartMetaLayerMediaType = "application/vnd.cncf.helm.chart.meta.layer.v1+json"
// HelmChartContentLayerMediaType is the reserved media type for Helm chart package content
HelmChartContentLayerMediaType = "application/vnd.cncf.helm.chart.content.layer.v1+tar"
// HelmChartMetaFileName is the reserved file name for Helm chart metadata
HelmChartMetaFileName = "chart-meta.json"
// HelmChartContentFileName is the reserved file name for Helm chart package content
HelmChartContentFileName = "chart-content.tgz"
// HelmChartNameAnnotation is the reserved annotation key for Helm chart name
HelmChartNameAnnotation = "sh.helm.chart.name"
// HelmChartVersionAnnotation is the reserved annotation key for Helm chart version
HelmChartVersionAnnotation = "sh.helm.chart.version"
)
// KnownMediaTypes returns a list of layer mediaTypes that the Helm client knows about
func KnownMediaTypes() []string {
return []string{
HelmChartMetaLayerMediaType,
HelmChartConfigMediaType,
HelmChartContentLayerMediaType,
}
}

@ -24,6 +24,6 @@ import (
func TestConstants(t *testing.T) {
knownMediaTypes := KnownMediaTypes()
assert.Contains(t, knownMediaTypes, HelmChartMetaLayerMediaType)
assert.Contains(t, knownMediaTypes, HelmChartConfigMediaType)
assert.Contains(t, knownMediaTypes, HelmChartContentLayerMediaType)
}

@ -18,6 +18,7 @@ package registry // import "helm.sh/helm/internal/experimental/registry"
import (
"errors"
"fmt"
"regexp"
"strings"
@ -59,6 +60,14 @@ func ParseReference(s string) (*Reference, error) {
return &ref, nil
}
// FullName the full name of a reference (repo:tag)
func (ref *Reference) FullName() string {
if ref.Tag == "" {
return ref.Repo
}
return fmt.Sprintf("%s:%s", ref.Repo, ref.Tag)
}
// setExtraFields adds the Repo and Tag fields to a Reference
func (ref *Reference) setExtraFields() {
ref.Tag = ref.Object

@ -44,46 +44,54 @@ func TestParseReference(t *testing.T) {
is.NoError(err)
is.Equal("mychart", ref.Repo)
is.Equal("", ref.Tag)
is.Equal("mychart", ref.FullName())
s = "mychart:1.5.0"
ref, err = ParseReference(s)
is.NoError(err)
is.Equal("mychart", ref.Repo)
is.Equal("1.5.0", ref.Tag)
is.Equal("mychart:1.5.0", ref.FullName())
s = "myrepo/mychart"
ref, err = ParseReference(s)
is.NoError(err)
is.Equal("myrepo/mychart", ref.Repo)
is.Equal("", ref.Tag)
is.Equal("myrepo/mychart", ref.FullName())
s = "myrepo/mychart:1.5.0"
ref, err = ParseReference(s)
is.NoError(err)
is.Equal("myrepo/mychart", ref.Repo)
is.Equal("1.5.0", ref.Tag)
is.Equal("myrepo/mychart:1.5.0", ref.FullName())
s = "mychart:5001:1.5.0"
ref, err = ParseReference(s)
is.NoError(err)
is.Equal("mychart:5001", ref.Repo)
is.Equal("1.5.0", ref.Tag)
is.Equal("mychart:5001:1.5.0", ref.FullName())
s = "myrepo:5001/mychart:1.5.0"
ref, err = ParseReference(s)
is.NoError(err)
is.Equal("myrepo:5001/mychart", ref.Repo)
is.Equal("1.5.0", ref.Tag)
is.Equal("myrepo:5001/mychart:1.5.0", ref.FullName())
s = "localhost:5000/mychart:latest"
ref, err = ParseReference(s)
is.NoError(err)
is.Equal("localhost:5000/mychart", ref.Repo)
is.Equal("latest", ref.Tag)
is.Equal("localhost:5000/mychart:latest", ref.FullName())
s = "my.host.com/my/nested/repo:1.2.3"
ref, err = ParseReference(s)
is.NoError(err)
is.Equal("my.host.com/my/nested/repo", ref.Repo)
is.Equal("1.2.3", ref.Tag)
is.Equal("my.host.com/my/nested/repo:1.2.3", ref.FullName())
}

@ -0,0 +1,66 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry // import "helm.sh/helm/internal/experimental/registry"
import (
"context"
"fmt"
"io"
"time"
orascontext "github.com/deislabs/oras/pkg/context"
units "github.com/docker/go-units"
"github.com/sirupsen/logrus"
)
// byteCountBinary produces a human-readable file size
func byteCountBinary(b int64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
}
// shortDigest returns first 7 characters of a sha256 digest
func shortDigest(digest string) string {
if len(digest) == 64 {
return digest[:7]
}
return digest
}
// timeAgo returns a human-readable timestamp respresenting time that has passed
func timeAgo(t time.Time) string {
return units.HumanDuration(time.Now().UTC().Sub(t))
}
// ctx retrieves a fresh context.
// disable verbose logging coming from ORAS (unless debug is enabled)
func ctx(out io.Writer, debug bool) context.Context {
if !debug {
return orascontext.Background()
}
ctx := orascontext.WithLoggerFromWriter(context.Background(), out)
orascontext.GetLogger(ctx).Logger.SetLevel(logrus.DebugLevel)
return ctx
}

@ -19,6 +19,7 @@ import (
"context"
"flag"
"io/ioutil"
"path/filepath"
"testing"
"time"
@ -54,20 +55,32 @@ func actionConfigFixture(t *testing.T) *Configuration {
t.Fatal(err)
}
return &Configuration{
Releases: storage.Init(driver.NewMemory()),
KubeClient: &kubefake.FailingKubeClient{PrintingKubeClient: kubefake.PrintingKubeClient{Out: ioutil.Discard}},
Capabilities: chartutil.DefaultCapabilities,
RegistryClient: registry.NewClient(&registry.ClientOptions{
Out: ioutil.Discard,
Authorizer: registry.Authorizer{
Client: client,
},
Resolver: registry.Resolver{
Resolver: resolver,
},
CacheRootDir: tdir,
cache, err := registry.NewCache(
registry.CacheOptDebug(true),
registry.CacheOptRoot(filepath.Join(tdir, registry.CacheRootDir)),
)
if err != nil {
t.Fatal(err)
}
registryClient, err := registry.NewClient(
registry.ClientOptAuthorizer(&registry.Authorizer{
Client: client,
}),
registry.ClientOptResolver(&registry.Resolver{
Resolver: resolver,
}),
registry.ClientOptCache(cache),
)
if err != nil {
t.Fatal(err)
}
return &Configuration{
Releases: storage.Init(driver.NewMemory()),
KubeClient: &kubefake.FailingKubeClient{PrintingKubeClient: kubefake.PrintingKubeClient{Out: ioutil.Discard}},
Capabilities: chartutil.DefaultCapabilities,
RegistryClient: registryClient,
Log: func(format string, v ...interface{}) {
t.Helper()
if *verbose {

Loading…
Cancel
Save