diff --git a/cmd/helm/dependency.go b/cmd/helm/dependency.go new file mode 100644 index 000000000..2da344052 --- /dev/null +++ b/cmd/helm/dependency.go @@ -0,0 +1,216 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + + "github.com/gosuri/uitable" + "github.com/spf13/cobra" + + "k8s.io/helm/pkg/chartutil" +) + +const dependencyDesc = ` +Manage the dependencies of a chart. + +Helm charts store their dependencies in 'charts/'. For chart developers, it is +often easier to manage a single dependency file ('requirements.yaml') +which declares all dependencies. + +The dependency commands operate on that file, making it easy to synchronize +between the desired dependencies and the actual dependencies stored in the +'charts/' directory. + +A 'requirements.yaml' file is a YAML file in which developers can declare chart +dependencies, along with the location of the chart and the desired version. +For example, this requirements file declares two dependencies: + + # requirements.yaml + dependencies: + - name: nginx + version: "1.2.3" + repository: "https://example.com/charts" + - name: memcached + version: "3.2.1" + repository: "https://another.example.com/charts" + +The 'name' should be the name of a chart, where that name must match the name +in that chart's 'Chart.yaml' file. + +The 'version' field should contain a semantic version or version range. + +The 'repository' URL should point to a Chart Repository. Helm expects that by +appending '/index.yaml' to the URL, it should be able to retrieve the chart +repository's index. Note: 'repository' cannot be a repository alias. It must be +a URL. +` + +const dependencyListDesc = ` +List all of the dependencies declared in a chart. + +This can take chart archives and chart directories as input. It will not alter +the contents of a chart. + +This will produce an error if the chart cannot be loaded. It will emit a warning +if it cannot find a requirements.yaml. +` + +func newDependencyCmd(out io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "dependency update|list", + Aliases: []string{"dep", "dependencies"}, + Short: "manage a chart's dependencies", + Long: dependencyDesc, + } + + cmd.AddCommand(newDependencyListCmd(out)) + cmd.AddCommand(newDependencyUpdateCmd(out)) + + return cmd +} + +type dependencyListCmd struct { + out io.Writer + chartpath string +} + +func newDependencyListCmd(out io.Writer) *cobra.Command { + dlc := &dependencyListCmd{ + out: out, + } + cmd := &cobra.Command{ + Use: "list [flags] CHART", + Aliases: []string{"ls"}, + Short: "list the dependencies for the given chart", + Long: dependencyListDesc, + RunE: func(cmd *cobra.Command, args []string) error { + cp := "." + if len(args) > 0 { + cp = args[0] + } + + var err error + dlc.chartpath, err = filepath.Abs(cp) + if err != nil { + return err + } + return dlc.run() + }, + } + return cmd +} + +func (l *dependencyListCmd) run() error { + c, err := chartutil.Load(l.chartpath) + if err != nil { + return err + } + + r, err := chartutil.LoadRequirements(c) + if err != nil { + if err == chartutil.ErrRequirementsNotFound { + fmt.Fprintf(l.out, "WARNING: no requirements at %s/charts", l.chartpath) + return nil + } + return err + } + + l.printRequirements(r, l.out) + fmt.Fprintln(l.out) + l.printMissing(r, l.out) + return nil +} + +func (l *dependencyListCmd) dependencyStatus(dep *chartutil.Dependency) string { + filename := fmt.Sprintf("%s-%s.tgz", dep.Name, dep.Version) + archive := filepath.Join(l.chartpath, "charts", filename) + if _, err := os.Stat(archive); err == nil { + c, err := chartutil.Load(archive) + if err != nil { + return "corrupt" + } + if c.Metadata.Name == dep.Name && c.Metadata.Version == dep.Version { + return "ok" + } + return "mismatch" + } + + folder := filepath.Join(l.chartpath, "charts", dep.Name) + if fi, err := os.Stat(folder); err != nil { + return "missing" + } else if !fi.IsDir() { + return "mispackaged" + } + + c, err := chartutil.Load(folder) + if err != nil { + return "corrupt" + } + + if c.Metadata.Name != dep.Name { + return "misnamed" + } + + if c.Metadata.Version != dep.Version { + return "wrong version" + } + + return "unpacked" +} + +// printRequirements prints all of the requirements in the yaml file. +func (l *dependencyListCmd) printRequirements(reqs *chartutil.Requirements, out io.Writer) { + table := uitable.New() + table.MaxColWidth = 80 + table.AddRow("NAME", "VERSION", "REPOSITORY", "STATUS") + for _, row := range reqs.Dependencies { + table.AddRow(row.Name, row.Version, row.Repository, l.dependencyStatus(row)) + } + fmt.Fprintln(out, table) +} + +// printMissing prints warnings about charts that are present on disk, but are not in the requirements. +func (l *dependencyListCmd) printMissing(reqs *chartutil.Requirements, out io.Writer) { + folder := filepath.Join(l.chartpath, "charts/*") + files, err := filepath.Glob(folder) + if err != nil { + fmt.Fprintln(l.out, err) + return + } + + for _, f := range files { + c, err := chartutil.Load(f) + if err != nil { + fmt.Fprintf(l.out, "WARNING: %q is not a chart.\n", f) + continue + } + found := false + for _, d := range reqs.Dependencies { + if d.Name == c.Metadata.Name { + found = true + break + } + } + if !found { + fmt.Fprintf(l.out, "WARNING: %q is not in requirements.yaml.\n", f) + } + } + +} diff --git a/cmd/helm/dependency_test.go b/cmd/helm/dependency_test.go new file mode 100644 index 000000000..749d490cb --- /dev/null +++ b/cmd/helm/dependency_test.go @@ -0,0 +1,71 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "strings" + "testing" +) + +func TestDependencyListCmd(t *testing.T) { + + tests := []struct { + name string + args []string + expect string + err bool + }{ + { + name: "No such chart", + args: []string{"/no/such/chart"}, + err: true, + }, + { + name: "No requirements.yaml", + args: []string{"testdata/testcharts/alpine"}, + expect: "WARNING: no requirements at ", + }, + { + name: "Requirements in chart dir", + args: []string{"testdata/testcharts/reqtest"}, + expect: "NAME \tVERSION\tREPOSITORY \tSTATUS \nreqsubchart \t0.1.0 \thttps://example.com/charts\tunpacked\nreqsubchart2\t0.2.0 \thttps://example.com/charts\tunpacked\n", + }, + { + name: "Requirements in chart archive", + args: []string{"testdata/testcharts/reqtest-0.1.0.tgz"}, + expect: "NAME \tVERSION\tREPOSITORY \tSTATUS \nreqsubchart \t0.1.0 \thttps://example.com/charts\tmissing\nreqsubchart2\t0.2.0 \thttps://example.com/charts\tmissing\n", + }, + } + + for _, tt := range tests { + buf := bytes.NewBuffer(nil) + dlc := newDependencyListCmd(buf) + if err := dlc.RunE(dlc, tt.args); err != nil { + if tt.err { + continue + } + t.Errorf("Test %q: %s", tt.name, err) + continue + } + + got := buf.String() + if !strings.Contains(got, tt.expect) { + t.Errorf("Test: %q, Expected:\n%q\nGot:\n%q", tt.name, tt.expect, got) + } + } + +} diff --git a/cmd/helm/dependency_update.go b/cmd/helm/dependency_update.go new file mode 100644 index 000000000..9b9eb1d12 --- /dev/null +++ b/cmd/helm/dependency_update.go @@ -0,0 +1,287 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/ghodss/yaml" + "github.com/spf13/cobra" + + "k8s.io/helm/cmd/helm/resolver" + "k8s.io/helm/pkg/chartutil" + "k8s.io/helm/pkg/repo" +) + +const dependencyUpDesc = ` +Update the on-disk dependencies to mirror the requirements.yaml file. + +This command verifies that the required charts, as expressed in 'requirements.yaml', +are present in 'charts/' and are at an acceptable version. +` + +// dependencyUpdateCmd describes a 'helm dependency update' +type dependencyUpdateCmd struct { + out io.Writer + chartpath string + repoFile string + repopath string + helmhome string + verify bool + keyring string +} + +// newDependencyUpdateCmd creates a new dependency update command. +func newDependencyUpdateCmd(out io.Writer) *cobra.Command { + duc := &dependencyUpdateCmd{ + out: out, + } + + cmd := &cobra.Command{ + Use: "update [flags] CHART", + Aliases: []string{"up"}, + Short: "update charts/ based on the contents of requirements.yaml", + Long: dependencyUpDesc, + RunE: func(cmd *cobra.Command, args []string) error { + cp := "." + if len(args) > 0 { + cp = args[0] + } + + var err error + duc.chartpath, err = filepath.Abs(cp) + if err != nil { + return err + } + + duc.helmhome = homePath() + duc.repoFile = repositoriesFile() + duc.repopath = repositoryDirectory() + + return duc.run() + }, + } + + f := cmd.Flags() + f.BoolVar(&duc.verify, "verify", false, "Verify the package against its signature.") + f.StringVar(&duc.keyring, "keyring", defaultKeyring(), "The keyring containing public keys.") + + return cmd +} + +// run runs the full dependency update process. +func (d *dependencyUpdateCmd) run() error { + if fi, err := os.Stat(d.chartpath); err != nil { + return fmt.Errorf("could not find %s: %s", d.chartpath, err) + } else if !fi.IsDir() { + return errors.New("only unpacked charts can be updated") + } + c, err := chartutil.LoadDir(d.chartpath) + if err != nil { + return err + } + + req, err := chartutil.LoadRequirements(c) + if err != nil { + if err == chartutil.ErrRequirementsNotFound { + fmt.Fprintf(d.out, "No requirements found in %s/charts.\n", d.chartpath) + return nil + } + return err + } + + // For each repo in the file, update the cached copy of that repo + if _, err := d.updateRepos(req.Dependencies); err != nil { + return err + } + + // Now we need to find out which version of a chart best satisfies the + // requirements the requirements.yaml + lock, err := d.resolve(req) + if err != nil { + return err + } + + // Now we need to fetch every package here into charts/ + if err := d.downloadAll(lock.Dependencies); err != nil { + return err + } + + // Finally, we need to write the lockfile. + return writeLock(d.chartpath, lock) +} + +// resolve takes a list of requirements and translates them into an exact version to download. +// +// This returns a lock file, which has all of the requirements normalized to a specific version. +func (d *dependencyUpdateCmd) resolve(req *chartutil.Requirements) (*chartutil.RequirementsLock, error) { + res := resolver.New(d.chartpath, d.helmhome) + return res.Resolve(req) +} + +// downloadAll takes a list of dependencies and downloads them into charts/ +func (d *dependencyUpdateCmd) downloadAll(deps []*chartutil.Dependency) error { + repos, err := loadChartRepositories(d.repopath) + if err != nil { + return err + } + + fmt.Fprintf(d.out, "Saving %d charts\n", len(deps)) + for _, dep := range deps { + fmt.Fprintf(d.out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) + + target := fmt.Sprintf("%s-%s", dep.Name, dep.Version) + churl, err := findChartURL(target, dep.Repository, repos) + if err != nil { + fmt.Fprintf(d.out, "WARNING: %s (skipped)", err) + continue + } + + dest := filepath.Join(d.chartpath, "charts", target+".tgz") + data, err := downloadChart(churl, d.verify, d.keyring) + if err != nil { + fmt.Fprintf(d.out, "WARNING: Could not download %s: %s (skipped)", churl, err) + continue + } + if err := ioutil.WriteFile(dest, data.Bytes(), 0655); err != nil { + fmt.Fprintf(d.out, "WARNING: %s (skipped)", err) + continue + } + } + return nil +} + +// updateRepos updates all of the local repos to their latest. +// +// If one of the dependencies present is not in the cached repos, this will error out. The +// consequence of that is that every repository referenced in a requirements.yaml file +// must also be added with 'helm repo add'. +func (d *dependencyUpdateCmd) updateRepos(deps []*chartutil.Dependency) (*repo.RepoFile, error) { + // TODO: In the future, we could make it so that only the repositories that + // are used by this chart are updated. As it is, we're mainly doing some sanity + // checking here. + rf, err := repo.LoadRepositoriesFile(d.repoFile) + if err != nil { + return rf, err + } + repos := rf.Repositories + + // Verify that all repositories referenced in the deps are actually known + // by Helm. + missing := []string{} + for _, dd := range deps { + found := false + if dd.Repository == "" { + found = true + } else { + for _, repo := range repos { + if urlsAreEqual(repo, dd.Repository) { + found = true + } + } + } + if !found { + missing = append(missing, dd.Repository) + } + } + + if len(missing) > 0 { + return rf, fmt.Errorf("no repository definition for %s. Try 'helm repo add'", strings.Join(missing, ", ")) + } + + if len(repos) > 0 { + // This prints errors straight to out. + updateCharts(repos, flagDebug, d.out) + } + return rf, nil +} + +// urlsAreEqual normalizes two URLs and then compares for equality. +func urlsAreEqual(a, b string) bool { + au, err := url.Parse(a) + if err != nil { + return a == b + } + bu, err := url.Parse(b) + if err != nil { + return false + } + return au.String() == bu.String() +} + +// findChartURL searches the cache of repo data for a chart that has the name and the repourl specified. +// +// In this current version, name is of the form 'foo-1.2.3'. This will change when +// the repository index stucture changes. +func findChartURL(name, repourl string, repos map[string]*repo.ChartRepository) (string, error) { + for _, cr := range repos { + if urlsAreEqual(repourl, cr.URL) { + for ename, entry := range cr.IndexFile.Entries { + if ename == name { + return entry.URL, nil + } + } + } + } + return "", fmt.Errorf("chart %s not found in %s", name, repourl) +} + +// loadChartRepositories reads the repositories.yaml, and then builds a map of +// ChartRepositories. +// +// The key is the local name (which is only present in the repositories.yaml). +func loadChartRepositories(repodir string) (map[string]*repo.ChartRepository, error) { + indices := map[string]*repo.ChartRepository{} + repoyaml := repositoriesFile() + + // Load repositories.yaml file + rf, err := repo.LoadRepositoriesFile(repoyaml) + if err != nil { + return indices, fmt.Errorf("failed to load %s: %s", repoyaml, err) + } + + // localName: chartRepo + for lname, url := range rf.Repositories { + index, err := repo.LoadIndexFile(cacheIndexFile(lname)) + if err != nil { + return indices, err + } + + cr := &repo.ChartRepository{ + URL: url, + IndexFile: index, + } + indices[lname] = cr + } + return indices, nil +} + +// writeLock writes a lockfile to disk +func writeLock(chartpath string, lock *chartutil.RequirementsLock) error { + data, err := yaml.Marshal(lock) + if err != nil { + return err + } + dest := filepath.Join(chartpath, "requirements.lock") + return ioutil.WriteFile(dest, data, 0755) +} diff --git a/cmd/helm/dependency_update_test.go b/cmd/helm/dependency_update_test.go new file mode 100644 index 000000000..b38b90a21 --- /dev/null +++ b/cmd/helm/dependency_update_test.go @@ -0,0 +1,214 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ghodss/yaml" + + "k8s.io/helm/pkg/chartutil" + "k8s.io/helm/pkg/proto/hapi/chart" + "k8s.io/helm/pkg/provenance" + "k8s.io/helm/pkg/repo" +) + +func TestDependencyUpdateCmd(t *testing.T) { + // Set up a testing helm home + oldhome := helmHome + hh, err := tempHelmHome() + if err != nil { + t.Fatal(err) + } + helmHome = hh // Shoot me now. + defer func() { + os.RemoveAll(hh) + helmHome = oldhome + }() + + srv := newTestingRepositoryServer(hh) + defer srv.stop() + copied, err := srv.copyCharts("testdata/testcharts/*.tgz") + t.Logf("Copied charts %s", strings.Join(copied, "\n")) + t.Logf("Listening for directory %s", srv.docroot) + + chartname := "depup" + if err := createTestingChart(hh, chartname, srv.url()); err != nil { + t.Fatal(err) + } + + out := bytes.NewBuffer(nil) + duc := &dependencyUpdateCmd{out: out} + duc.helmhome = hh + duc.chartpath = filepath.Join(hh, chartname) + duc.repoFile = filepath.Join(duc.helmhome, "repository/repositories.yaml") + duc.repopath = filepath.Join(duc.helmhome, "repository") + + if err := duc.run(); err != nil { + t.Fatal(err) + } + + output := out.String() + t.Logf("Output: %s", output) + // This is written directly to stdout, so we have to capture as is. + if !strings.Contains(output, `update from the "test" chart repository`) { + t.Errorf("Repo did not get updated\n%s", output) + } + + // Make sure the actual file got downloaded. + expect := filepath.Join(hh, chartname, "charts/reqtest-0.1.0.tgz") + if _, err := os.Stat(expect); err != nil { + t.Fatal(err) + } + + hash, err := provenance.DigestFile(expect) + if err != nil { + t.Fatal(err) + } + + i, err := repo.LoadIndexFile(cacheIndexFile("test")) + if err != nil { + t.Fatal(err) + } + + if h := i.Entries["reqtest-0.1.0"].Digest; h != hash { + t.Errorf("Failed hash match: expected %s, got %s", hash, h) + } + + t.Logf("Results: %s", out.String()) +} + +// newTestingRepositoryServer creates a repository server for testing. +// +// docroot should be a temp dir managed by the caller. +// +// This will start the server, serving files off of the docroot. +// +// Use copyCharts to move charts into the repository and then index them +// for service. +func newTestingRepositoryServer(docroot string) *testingRepositoryServer { + root, err := filepath.Abs(docroot) + if err != nil { + panic(err) + } + srv := &testingRepositoryServer{ + docroot: root, + } + srv.start() + // Add the testing repository as the only repo. + if err := setTestingRepository(docroot, "test", srv.url()); err != nil { + panic(err) + } + return srv +} + +type testingRepositoryServer struct { + docroot string + srv *httptest.Server +} + +// copyCharts takes a glob expression and copies those charts to the server root. +func (s *testingRepositoryServer) copyCharts(origin string) ([]string, error) { + files, err := filepath.Glob(origin) + if err != nil { + return []string{}, err + } + copied := make([]string, len(files)) + for i, f := range files { + base := filepath.Base(f) + newname := filepath.Join(s.docroot, base) + data, err := ioutil.ReadFile(f) + if err != nil { + return []string{}, err + } + if err := ioutil.WriteFile(newname, data, 0755); err != nil { + return []string{}, err + } + copied[i] = newname + } + + // generate the index + index, err := repo.IndexDirectory(s.docroot, s.url()) + if err != nil { + return copied, err + } + + d, err := yaml.Marshal(index.Entries) + if err != nil { + return copied, err + } + + ifile := filepath.Join(s.docroot, "index.yaml") + err = ioutil.WriteFile(ifile, d, 0755) + return copied, err +} + +func (s *testingRepositoryServer) start() { + s.srv = httptest.NewServer(http.FileServer(http.Dir(s.docroot))) +} + +func (s *testingRepositoryServer) stop() { + s.srv.Close() +} + +func (s *testingRepositoryServer) url() string { + return s.srv.URL +} + +// setTestingRepository sets up a testing repository.yaml with only the given name/URL. +func setTestingRepository(helmhome, name, url string) error { + // Oddly, there is no repo.Save function for this. + data, err := yaml.Marshal(&map[string]string{name: url}) + if err != nil { + return err + } + os.MkdirAll(filepath.Join(helmhome, "repository", name), 0755) + dest := filepath.Join(helmhome, "repository/repositories.yaml") + return ioutil.WriteFile(dest, data, 0666) +} + +// createTestingChart creates a basic chart that depends on reqtest-0.1.0 +// +// The baseURL can be used to point to a particular repository server. +func createTestingChart(dest, name, baseURL string) error { + cfile := &chart.Metadata{ + Name: name, + Version: "1.2.3", + } + dir := filepath.Join(dest, name) + _, err := chartutil.Create(cfile, dest) + if err != nil { + return err + } + req := &chartutil.Requirements{ + Dependencies: []*chartutil.Dependency{ + {Name: "reqtest", Version: "0.1.0", Repository: baseURL}, + }, + } + data, err := yaml.Marshal(req) + if err != nil { + return err + } + + return ioutil.WriteFile(filepath.Join(dir, "requirements.yaml"), data, 0655) +} diff --git a/cmd/helm/fetch.go b/cmd/helm/fetch.go index 395fef175..e174bcdac 100644 --- a/cmd/helm/fetch.go +++ b/cmd/helm/fetch.go @@ -96,28 +96,36 @@ func (f *fetchCmd) run() error { pname += ".tgz" } - return downloadChart(pname, f.untar, f.untardir, f.verify, f.keyring) + return downloadAndSaveChart(pname, f.untar, f.untardir, f.verify, f.keyring) } -// downloadChart fetches a chart over HTTP, and then (if verify is true) verifies it. +// downloadAndSaveChart fetches a chart over HTTP, and then (if verify is true) verifies it. // // If untar is true, it also unpacks the file into untardir. -func downloadChart(pname string, untar bool, untardir string, verify bool, keyring string) error { - r, err := repo.LoadRepositoriesFile(repositoriesFile()) +func downloadAndSaveChart(pname string, untar bool, untardir string, verify bool, keyring string) error { + buf, err := downloadChart(pname, verify, keyring) if err != nil { return err } + return saveChart(pname, buf, untar, untardir) +} + +func downloadChart(pname string, verify bool, keyring string) (*bytes.Buffer, error) { + r, err := repo.LoadRepositoriesFile(repositoriesFile()) + if err != nil { + return bytes.NewBuffer(nil), err + } // get download url u, err := mapRepoArg(pname, r.Repositories) if err != nil { - return err + return bytes.NewBuffer(nil), err } href := u.String() buf, err := fetchChart(href) if err != nil { - return err + return buf, err } if verify { @@ -125,17 +133,17 @@ func downloadChart(pname string, untar bool, untardir string, verify bool, keyri sigref := href + ".prov" sig, err := fetchChart(sigref) if err != nil { - return fmt.Errorf("provenance data not downloaded from %s: %s", sigref, err) + return buf, fmt.Errorf("provenance data not downloaded from %s: %s", sigref, err) } if err := ioutil.WriteFile(basename+".prov", sig.Bytes(), 0755); err != nil { - return fmt.Errorf("provenance data not saved: %s", err) + return buf, fmt.Errorf("provenance data not saved: %s", err) } if err := verifyChart(basename, keyring); err != nil { - return err + return buf, err } } - return saveChart(pname, buf, untar, untardir) + return buf, nil } // verifyChart takes a path to a chart archive and a keyring, and verifies the chart. diff --git a/cmd/helm/helm.go b/cmd/helm/helm.go index 78e058b02..76d53ca21 100644 --- a/cmd/helm/helm.go +++ b/cmd/helm/helm.go @@ -99,6 +99,7 @@ func newRootCmd(out io.Writer) *cobra.Command { newVerifyCmd(out), newUpdateCmd(out), newVersionCmd(nil, out), + newDependencyCmd(out), ) return cmd } diff --git a/cmd/helm/helm_test.go b/cmd/helm/helm_test.go index 68c3ccc98..e0ce52c74 100644 --- a/cmd/helm/helm_test.go +++ b/cmd/helm/helm_test.go @@ -20,6 +20,7 @@ import ( "bytes" "fmt" "io" + "io/ioutil" "math/rand" "regexp" "testing" @@ -205,3 +206,22 @@ type releaseCase struct { err bool resp *release.Release } + +// tmpHelmHome sets up a Helm Home in a temp dir. +// +// This does not clean up the directory. You must do that yourself. +// You must also set helmHome yourself. +func tempHelmHome() (string, error) { + oldhome := helmHome + dir, err := ioutil.TempDir("", "helm_home-") + if err != nil { + return "n/", err + } + + helmHome = dir + if err := ensureHome(); err != nil { + return "n/", err + } + helmHome = oldhome + return dir, nil +} diff --git a/cmd/helm/init_test.go b/cmd/helm/init_test.go index 44bcafce2..df20e3786 100644 --- a/cmd/helm/init_test.go +++ b/cmd/helm/init_test.go @@ -28,7 +28,7 @@ import ( func TestEnsureHome(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") - fmt.Fprintln(w, "OK") + fmt.Fprintln(w, "") })) defaultRepositoryURL = ts.URL diff --git a/cmd/helm/install.go b/cmd/helm/install.go index ced11eb9c..3fab2c7fd 100644 --- a/cmd/helm/install.go +++ b/cmd/helm/install.go @@ -306,7 +306,7 @@ func locateChartPath(name string, verify bool, keyring string) (string, error) { if filepath.Ext(name) != ".tgz" { name += ".tgz" } - if err := downloadChart(name, false, ".", verify, keyring); err == nil { + if err := downloadAndSaveChart(name, false, ".", verify, keyring); err == nil { lname, err := filepath.Abs(filepath.Base(name)) if err != nil { return lname, err diff --git a/cmd/helm/repo_test.go b/cmd/helm/repo_test.go index d24f6f594..1e64719e6 100644 --- a/cmd/helm/repo_test.go +++ b/cmd/helm/repo_test.go @@ -36,7 +36,7 @@ var ( func TestRepoAdd(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") - fmt.Fprintln(w, "OK") + fmt.Fprintln(w, "") })) helmHome, _ = ioutil.TempDir("", "helm_home") diff --git a/cmd/helm/resolver/resolver.go b/cmd/helm/resolver/resolver.go new file mode 100644 index 000000000..34854b538 --- /dev/null +++ b/cmd/helm/resolver/resolver.go @@ -0,0 +1,87 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resolver + +import ( + "bytes" + "encoding/json" + "fmt" + "time" + + "github.com/Masterminds/semver" + + "k8s.io/helm/pkg/chartutil" + "k8s.io/helm/pkg/provenance" +) + +// Resolver resolves dependencies from semantic version ranges to a particular version. +type Resolver struct { + chartpath string + helmhome string +} + +// New creates a new resolver for a given chart and a given helm home. +func New(chartpath string, helmhome string) *Resolver { + return &Resolver{ + chartpath: chartpath, + helmhome: helmhome, + } +} + +// Resolve resolves dependencies and returns a lock file with the resolution. +func (r *Resolver) Resolve(reqs *chartutil.Requirements) (*chartutil.RequirementsLock, error) { + d, err := hashReq(reqs) + if err != nil { + return nil, err + } + + // Now we clone the dependencies, locking as we go. + locked := make([]*chartutil.Dependency, len(reqs.Dependencies)) + for i, d := range reqs.Dependencies { + // Right now, we're just copying one entry to another. What we need to + // do here is parse the requirement as a SemVer range, and then look up + // whether a version in index.yaml satisfies this constraint. If so, + // we need to clone the dep, settinv Version appropriately. + // If not, we need to error out. + if _, err := semver.NewVersion(d.Version); err != nil { + return nil, fmt.Errorf("dependency %q has an invalid version: %s", d.Name, err) + } + locked[i] = &chartutil.Dependency{ + Name: d.Name, + Repository: d.Repository, + Version: d.Version, + } + } + + return &chartutil.RequirementsLock{ + Generated: time.Now(), + Digest: d, + Dependencies: locked, + }, nil +} + +// hashReq generates a hash of the requirements. +// +// This should be used only to compare against another hash generated by this +// function. +func hashReq(req *chartutil.Requirements) (string, error) { + data, err := json.Marshal(req) + if err != nil { + return "", err + } + s, err := provenance.Digest(bytes.NewBuffer(data)) + return "sha256:" + s, err +} diff --git a/cmd/helm/resolver/resolver_test.go b/cmd/helm/resolver/resolver_test.go new file mode 100644 index 000000000..0212757b8 --- /dev/null +++ b/cmd/helm/resolver/resolver_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resolver + +import ( + "testing" + + "k8s.io/helm/pkg/chartutil" +) + +func TestResolve(t *testing.T) { + tests := []struct { + name string + req *chartutil.Requirements + expect *chartutil.RequirementsLock + err bool + }{ + { + name: "version failure", + req: &chartutil.Requirements{ + Dependencies: []*chartutil.Dependency{ + {Name: "oedipus-rex", Repository: "http://example.com", Version: ">1"}, + }, + }, + err: true, + }, + { + name: "valid lock", + req: &chartutil.Requirements{ + Dependencies: []*chartutil.Dependency{ + {Name: "antigone", Repository: "http://example.com", Version: "1.0.0"}, + }, + }, + expect: &chartutil.RequirementsLock{ + Dependencies: []*chartutil.Dependency{ + {Name: "antigone", Repository: "http://example.com", Version: "1.0.0"}, + }, + }, + }, + } + + r := New("testdata/chartpath", "testdata/helmhome") + for _, tt := range tests { + l, err := r.Resolve(tt.req) + if err != nil { + if tt.err { + continue + } + t.Fatal(err) + } + + if tt.err { + t.Fatalf("Expected error in test %q", tt.name) + } + + if h, err := hashReq(tt.req); err != nil { + t.Fatal(err) + } else if h != l.Digest { + t.Errorf("%q: hashes don't match.", tt.name) + } + + // Check fields. + if len(l.Dependencies) != len(tt.req.Dependencies) { + t.Errorf("%s: wrong number of dependencies in lock", tt.name) + } + d0 := l.Dependencies[0] + e0 := tt.expect.Dependencies[0] + if d0.Name != e0.Name { + t.Errorf("%s: expected name %s, got %s", tt.name, e0.Name, d0.Name) + } + if d0.Repository != e0.Repository { + t.Errorf("%s: expected repo %s, got %s", tt.name, e0.Repository, d0.Repository) + } + if d0.Version != e0.Version { + t.Errorf("%s: expected version %s, got %s", tt.name, e0.Version, d0.Version) + } + } +} + +func TestHashReq(t *testing.T) { + expect := "sha256:e70e41f8922e19558a8bf62f591a8b70c8e4622e3c03e5415f09aba881f13885" + req := &chartutil.Requirements{ + Dependencies: []*chartutil.Dependency{ + {Name: "alpine", Version: "0.1.0", Repository: "http://localhost:8879/charts"}, + }, + } + h, err := hashReq(req) + if err != nil { + t.Fatal(err) + } + if expect != h { + t.Errorf("Expected %q, got %q", expect, h) + } + + req = &chartutil.Requirements{Dependencies: []*chartutil.Dependency{}} + h, err = hashReq(req) + if err != nil { + t.Fatal(err) + } + if expect == h { + t.Errorf("Expected %q != %q", expect, h) + } +} diff --git a/cmd/helm/search.go b/cmd/helm/search.go index 3c06aec47..e1a465c99 100644 --- a/cmd/helm/search.go +++ b/cmd/helm/search.go @@ -45,6 +45,7 @@ func search(cmd *cobra.Command, args []string) error { return errors.New("This command needs at least one argument (search string)") } + // TODO: This needs to be refactored to use loadChartRepositories results, err := searchCacheForPattern(cacheDirectory(), args[0]) if err != nil { return err diff --git a/cmd/helm/testdata/testcharts/reqtest-0.1.0.tgz b/cmd/helm/testdata/testcharts/reqtest-0.1.0.tgz new file mode 100644 index 000000000..356bc9303 Binary files /dev/null and b/cmd/helm/testdata/testcharts/reqtest-0.1.0.tgz differ diff --git a/cmd/helm/testdata/testcharts/reqtest/.helmignore b/cmd/helm/testdata/testcharts/reqtest/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/cmd/helm/testdata/testcharts/reqtest/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/cmd/helm/testdata/testcharts/reqtest/Chart.yaml b/cmd/helm/testdata/testcharts/reqtest/Chart.yaml new file mode 100755 index 000000000..e2fbe4b01 --- /dev/null +++ b/cmd/helm/testdata/testcharts/reqtest/Chart.yaml @@ -0,0 +1,3 @@ +description: A Helm chart for Kubernetes +name: reqtest +version: 0.1.0 diff --git a/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart/.helmignore b/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart/Chart.yaml b/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart/Chart.yaml new file mode 100755 index 000000000..c3813bc8c --- /dev/null +++ b/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart/Chart.yaml @@ -0,0 +1,3 @@ +description: A Helm chart for Kubernetes +name: reqsubchart +version: 0.1.0 diff --git a/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart/values.yaml b/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart/values.yaml new file mode 100644 index 000000000..0f0b63f2a --- /dev/null +++ b/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart/values.yaml @@ -0,0 +1,4 @@ +# Default values for reqsubchart. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value diff --git a/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart2/.helmignore b/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart2/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart2/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart2/Chart.yaml b/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart2/Chart.yaml new file mode 100755 index 000000000..9f7c22a71 --- /dev/null +++ b/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart2/Chart.yaml @@ -0,0 +1,3 @@ +description: A Helm chart for Kubernetes +name: reqsubchart2 +version: 0.2.0 diff --git a/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart2/values.yaml b/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart2/values.yaml new file mode 100644 index 000000000..0f0b63f2a --- /dev/null +++ b/cmd/helm/testdata/testcharts/reqtest/charts/reqsubchart2/values.yaml @@ -0,0 +1,4 @@ +# Default values for reqsubchart. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value diff --git a/cmd/helm/testdata/testcharts/reqtest/requirements.lock b/cmd/helm/testdata/testcharts/reqtest/requirements.lock new file mode 100755 index 000000000..ab1ae8cc0 --- /dev/null +++ b/cmd/helm/testdata/testcharts/reqtest/requirements.lock @@ -0,0 +1,3 @@ +dependencies: [] +digest: Not implemented +generated: 2016-09-13T17:25:17.593788787-06:00 diff --git a/cmd/helm/testdata/testcharts/reqtest/requirements.yaml b/cmd/helm/testdata/testcharts/reqtest/requirements.yaml new file mode 100644 index 000000000..4b0b8c2db --- /dev/null +++ b/cmd/helm/testdata/testcharts/reqtest/requirements.yaml @@ -0,0 +1,7 @@ +dependencies: + - name: reqsubchart + version: 0.1.0 + repository: "https://example.com/charts" + - name: reqsubchart2 + version: 0.2.0 + repository: "https://example.com/charts" diff --git a/cmd/helm/testdata/testcharts/reqtest/values.yaml b/cmd/helm/testdata/testcharts/reqtest/values.yaml new file mode 100644 index 000000000..d57f76b07 --- /dev/null +++ b/cmd/helm/testdata/testcharts/reqtest/values.yaml @@ -0,0 +1,4 @@ +# Default values for reqtest. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value diff --git a/docs/charts.md b/docs/charts.md index 104c3ad18..7b5e1ee67 100644 --- a/docs/charts.md +++ b/docs/charts.md @@ -125,6 +125,7 @@ chart's `charts/` directory: ``` wordpress: Chart.yaml + requirements.yaml # ... charts/ apache/ @@ -142,6 +143,61 @@ directory. **TIP:** _To drop a dependency into your `charts/` directory, use the `helm fetch` command._ +### Managing Dependencies with `requirements.yaml` + +While Helm will allow you to manually manage your dependencies, the +preferred method of declaring dependencies is by using a +`requirements.yaml` file inside of your chart. + +A `requirements.yaml` file is a simple file for listing your +dependencies. + +```yaml +dependencies: + - name: apache + version: 1.2.3 + repository: http://example.com/charts + - name: mysql + version: 3.2.1 + repository: http://another.example.com/charts +``` + +- The `name` field is the name of the chart you want. +- The `version` field is the version of the chart you want. +- The `repository` field is the full URL to the chart repository. Note + that you must also use `helm repo add` to add that repo locally. + +Once you have a dependencies file, you can run `helm dependency update` +and it will use your dependency file to download all of the specified +charts into your `charts/` directory for you. + +```console +$ helm dep up foochart +Hang tight while we grab the latest from your chart repositories... +...Successfully got an update from the "local" chart repository +...Successfully got an update from the "stable" chart repository +...Successfully got an update from the "example" chart repository +...Successfully got an update from the "another" chart repository +Update Complete. Happy Helming! +Saving 2 charts +Downloading apache from repo http://example.com/charts +Downloading mysql from repo http://another.example.com/charts +``` + +When `helm dependency update` retrieves charts, it will store them as +chart archives in the `charts/` directory. So for the example above, one +would expect to see the following files in the charts directory: + +``` +charts/ + apache-1.2.3.tgz + mysql-3.2.1.tgz +``` + +Manging charts with `requirements.yaml` is a good way to easily keep +charts updated, and also share requirements information throughout a +team. + ## Templates and Values By default, Helm Chart templates are written in the Go template language, with the diff --git a/pkg/chartutil/load_test.go b/pkg/chartutil/load_test.go index 822e8d078..3b785d20b 100644 --- a/pkg/chartutil/load_test.go +++ b/pkg/chartutil/load_test.go @@ -29,6 +29,7 @@ func TestLoadDir(t *testing.T) { } verifyFrobnitz(t, c) verifyChart(t, c) + verifyRequirements(t, c) } func TestLoadFile(t *testing.T) { @@ -38,6 +39,7 @@ func TestLoadFile(t *testing.T) { } verifyFrobnitz(t, c) verifyChart(t, c) + verifyRequirements(t, c) } func verifyChart(t *testing.T, c *chart.Chart) { @@ -49,7 +51,7 @@ func verifyChart(t *testing.T, c *chart.Chart) { t.Errorf("Expected 1 template, got %d", len(c.Templates)) } - numfiles := 6 + numfiles := 7 if len(c.Files) != numfiles { t.Errorf("Expected %d extra files, got %d", numfiles, len(c.Files)) for _, n := range c.Files { @@ -88,6 +90,32 @@ func verifyChart(t *testing.T, c *chart.Chart) { } +func verifyRequirements(t *testing.T, c *chart.Chart) { + r, err := LoadRequirements(c) + if err != nil { + t.Fatal(err) + } + if len(r.Dependencies) != 2 { + t.Errorf("Expected 2 requirements, got %d", len(r.Dependencies)) + } + tests := []*Dependency{ + {Name: "alpine", Version: "0.1.0", Repository: "https://example.com/charts"}, + {Name: "mariner", Version: "4.3.2", Repository: "https://example.com/charts"}, + } + for i, tt := range tests { + d := r.Dependencies[i] + if d.Name != tt.Name { + t.Errorf("Expected dependency named %q, got %q", tt.Name, d.Name) + } + if d.Version != tt.Version { + t.Errorf("Expected dependency named %q to have version %q, got %q", tt.Name, tt.Version, d.Version) + } + if d.Repository != tt.Repository { + t.Errorf("Expected dependency named %q to have repository %q, got %q", tt.Name, tt.Repository, d.Repository) + } + } +} + func verifyFrobnitz(t *testing.T, c *chart.Chart) { verifyChartfile(t, c.Metadata) diff --git a/pkg/chartutil/requirements.go b/pkg/chartutil/requirements.go new file mode 100644 index 000000000..4a452f64c --- /dev/null +++ b/pkg/chartutil/requirements.go @@ -0,0 +1,84 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chartutil + +import ( + "errors" + "time" + + "github.com/ghodss/yaml" + + "k8s.io/helm/pkg/proto/hapi/chart" +) + +// Dependency describes a chart upon which another chart depends. +// +// Dependencies can be used to express developer intent, or to capture the state +// of a chart. +type Dependency struct { + // Name is the name of the dependency. + // + // This must mach the name in the dependency's Chart.yaml. + Name string `json:"name"` + // Version is the version (range) of this chart. + // + // A lock file will always produce a single version, while a dependency + // may contain a semantic version range. + Version string `json:"version,omitempty"` + // The URL to the repository. + // + // Appending `index.yaml` to this string should result in a URL that can be + // used to fetch the repository index. + Repository string `json:"repository"` +} + +// Requirements is a list of requirements for a chart. +// +// Requirements are charts upon which this chart depends. This expresses +// developer intent. +type Requirements struct { + Dependencies []*Dependency `json:"dependencies"` +} + +// RequirementsLock is a lock file for requirements. +// +// It represents the state that the dependencies should be in. +type RequirementsLock struct { + // Genderated is the date the lock file was last generated. + Generated time.Time `json:"generated"` + // Digest is a hash of the requirements file used to generate it. + Digest string `json:"digest"` + // Dependencies is the list of dependencies that this lock file has locked. + Dependencies []*Dependency `json:"dependencies"` +} + +// ErrRequirementsNotFound indicates that a requirements.yaml is not found. +var ErrRequirementsNotFound = errors.New("requirements.yaml not found") + +// LoadRequirements loads a requirements file from an in-memory chart. +func LoadRequirements(c *chart.Chart) (*Requirements, error) { + var data []byte + for _, f := range c.Files { + if f.TypeUrl == "requirements.yaml" { + data = f.Value + } + } + if len(data) == 0 { + return nil, ErrRequirementsNotFound + } + r := &Requirements{} + return r, yaml.Unmarshal(data, r) +} diff --git a/pkg/chartutil/requirements_test.go b/pkg/chartutil/requirements_test.go new file mode 100644 index 000000000..04f1bd4a6 --- /dev/null +++ b/pkg/chartutil/requirements_test.go @@ -0,0 +1,27 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package chartutil + +import ( + "testing" +) + +func TestLoadRequirements(t *testing.T) { + c, err := Load("testdata/frobnitz") + if err != nil { + t.Fatalf("Failed to load testdata: %s", err) + } + verifyRequirements(t, c) +} diff --git a/pkg/chartutil/testdata/frobnitz-1.2.3.tgz b/pkg/chartutil/testdata/frobnitz-1.2.3.tgz index 8055a0094..27bec72bd 100644 Binary files a/pkg/chartutil/testdata/frobnitz-1.2.3.tgz and b/pkg/chartutil/testdata/frobnitz-1.2.3.tgz differ diff --git a/pkg/chartutil/testdata/frobnitz/charts/mariner-4.3.2.tgz b/pkg/chartutil/testdata/frobnitz/charts/mariner-4.3.2.tgz index 03de29d68..408f3bd5c 100644 Binary files a/pkg/chartutil/testdata/frobnitz/charts/mariner-4.3.2.tgz and b/pkg/chartutil/testdata/frobnitz/charts/mariner-4.3.2.tgz differ diff --git a/pkg/chartutil/testdata/frobnitz/requirements.yaml b/pkg/chartutil/testdata/frobnitz/requirements.yaml new file mode 100644 index 000000000..5eb0bc98b --- /dev/null +++ b/pkg/chartutil/testdata/frobnitz/requirements.yaml @@ -0,0 +1,7 @@ +dependencies: + - name: alpine + version: "0.1.0" + repository: https://example.com/charts + - name: mariner + version: "4.3.2" + repository: https://example.com/charts diff --git a/pkg/chartutil/testdata/mariner/charts/albatross-0.1.0.tgz b/pkg/chartutil/testdata/mariner/charts/albatross-0.1.0.tgz index 8b1b19dce..97e0b2eb8 100644 Binary files a/pkg/chartutil/testdata/mariner/charts/albatross-0.1.0.tgz and b/pkg/chartutil/testdata/mariner/charts/albatross-0.1.0.tgz differ diff --git a/pkg/chartutil/testdata/mariner/requirements.yaml b/pkg/chartutil/testdata/mariner/requirements.yaml new file mode 100644 index 000000000..0b21d15b7 --- /dev/null +++ b/pkg/chartutil/testdata/mariner/requirements.yaml @@ -0,0 +1,4 @@ +dependencies: + - name: albatross + repository: https://example.com/mariner/charts + version: "0.1.0" diff --git a/pkg/provenance/sign.go b/pkg/provenance/sign.go index ab0fa6ebe..ae4c6d2b6 100644 --- a/pkg/provenance/sign.go +++ b/pkg/provenance/sign.go @@ -204,7 +204,7 @@ func (s *Signatory) Verify(chartpath, sigpath string) (*Verification, error) { ver.SignedBy = by // Second, verify the hash of the tarball. - sum, err := sumArchive(chartpath) + sum, err := DigestFile(chartpath) if err != nil { return ver, err } @@ -254,7 +254,7 @@ func (s *Signatory) verifySignature(block *clearsign.Block) (*openpgp.Entity, er func messageBlock(chartpath string) (*bytes.Buffer, error) { var b *bytes.Buffer // Checksum the archive - chash, err := sumArchive(chartpath) + chash, err := DigestFile(chartpath) if err != nil { return b, err } @@ -332,20 +332,26 @@ func loadKeyRing(ringpath string) (openpgp.EntityList, error) { return openpgp.ReadKeyRing(f) } -// sumArchive calculates a SHA256 hash (like Docker) for a given file. +// DigestFile calculates a SHA256 hash (like Docker) for a given file. // // It takes the path to the archive file, and returns a string representation of // the SHA256 sum. // // The intended use of this function is to generate a sum of a chart TGZ file. -func sumArchive(filename string) (string, error) { +func DigestFile(filename string) (string, error) { f, err := os.Open(filename) if err != nil { return "", err } defer f.Close() + return Digest(f) +} +// Digest hashes a reader and returns a SHA256 digest. +// +// Helm uses SHA256 as its default hash for all non-cryptographic applications. +func Digest(in io.Reader) (string, error) { hash := crypto.SHA256.New() - io.Copy(hash, f) + io.Copy(hash, in) return hex.EncodeToString(hash.Sum(nil)), nil } diff --git a/pkg/provenance/sign_test.go b/pkg/provenance/sign_test.go index 2f66748c1..00bc5aced 100644 --- a/pkg/provenance/sign_test.go +++ b/pkg/provenance/sign_test.go @@ -127,6 +127,28 @@ func TestLoadKeyRing(t *testing.T) { } } +func TestDigest(t *testing.T) { + f, err := os.Open(testChartfile) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + hash, err := Digest(f) + if err != nil { + t.Fatal(err) + } + + sig, err := readSumFile(testSumfile) + if err != nil { + t.Fatal(err) + } + + if !strings.Contains(sig, hash) { + t.Errorf("Expected %s to be in %s", hash, sig) + } +} + func TestNewFromFiles(t *testing.T) { s, err := NewFromFiles(testKeyfile, testPubfile) if err != nil { @@ -138,8 +160,8 @@ func TestNewFromFiles(t *testing.T) { } } -func TestSumArchive(t *testing.T) { - hash, err := sumArchive(testChartfile) +func TestDigestFile(t *testing.T) { + hash, err := DigestFile(testChartfile) if err != nil { t.Fatal(err) } diff --git a/pkg/repo/index.go b/pkg/repo/index.go index 5e9a08837..b7aade1dc 100644 --- a/pkg/repo/index.go +++ b/pkg/repo/index.go @@ -19,11 +19,14 @@ package repo import ( "io/ioutil" "net/http" + "path/filepath" "strings" "gopkg.in/yaml.v2" + "k8s.io/helm/pkg/chartutil" "k8s.io/helm/pkg/proto/hapi/chart" + "k8s.io/helm/pkg/provenance" ) var indexPath = "index.yaml" @@ -33,14 +36,61 @@ type IndexFile struct { Entries map[string]*ChartRef } +// NewIndexFile initializes an index. +func NewIndexFile() *IndexFile { + return &IndexFile{Entries: map[string]*ChartRef{}} +} + +// Add adds a file to the index +func (i IndexFile) Add(md *chart.Metadata, filename, baseURL, digest string) { + name := strings.TrimSuffix(filename, ".tgz") + cr := &ChartRef{ + Name: name, + URL: baseURL + "/" + filename, + Chartfile: md, + Digest: digest, + // FIXME: Need to add Created + } + i.Entries[name] = cr +} + +// Need both JSON and YAML annotations until we get rid of gopkg.in/yaml.v2 + // ChartRef represents a chart entry in the IndexFile type ChartRef struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - Created string `yaml:"created,omitempty"` - Removed bool `yaml:"removed,omitempty"` - Digest string `yaml:"digest,omitempty"` - Chartfile *chart.Metadata `yaml:"chartfile"` + Name string `yaml:"name" json:"name"` + URL string `yaml:"url" json:"url"` + Created string `yaml:"created,omitempty" json:"created,omitempty"` + Removed bool `yaml:"removed,omitempty" json:"removed,omitempty"` + Digest string `yaml:"digest,omitempty" json:"digest,omitempty"` + Chartfile *chart.Metadata `yaml:"chartfile" json:"chartfile"` +} + +// IndexDirectory reads a (flat) directory and generates an index. +// +// It indexes only charts that have been packaged (*.tgz). +// +// It writes the results to dir/index.yaml. +func IndexDirectory(dir, baseURL string) (*IndexFile, error) { + archives, err := filepath.Glob(filepath.Join(dir, "*.tgz")) + if err != nil { + return nil, err + } + index := NewIndexFile() + for _, arch := range archives { + fname := filepath.Base(arch) + c, err := chartutil.Load(arch) + if err != nil { + // Assume this is not a chart. + continue + } + hash, err := provenance.DigestFile(arch) + if err != nil { + return index, err + } + index.Add(c.Metadata, fname, baseURL, hash) + } + return index, nil } // DownloadIndexFile uses @@ -72,9 +122,7 @@ func DownloadIndexFile(repoName, url, indexFilePath string) error { func (i *IndexFile) UnmarshalYAML(unmarshal func(interface{}) error) error { var refs map[string]*ChartRef if err := unmarshal(&refs); err != nil { - if _, ok := err.(*yaml.TypeError); !ok { - return err - } + return err } i.Entries = refs return nil @@ -101,11 +149,11 @@ func LoadIndexFile(path string) (*IndexFile, error) { return nil, err } - var indexfile IndexFile - err = yaml.Unmarshal(b, &indexfile) + indexfile := NewIndexFile() + err = yaml.Unmarshal(b, indexfile) if err != nil { return nil, err } - return &indexfile, nil + return indexfile, nil } diff --git a/pkg/repo/index_test.go b/pkg/repo/index_test.go index 4fd4c255b..3b248fb19 100644 --- a/pkg/repo/index_test.go +++ b/pkg/repo/index_test.go @@ -110,3 +110,35 @@ func TestLoadIndexFile(t *testing.T) { t.Errorf("alpine entry was not decoded properly") } } + +func TestIndexDirectory(t *testing.T) { + dir := "testdata/repository" + index, err := IndexDirectory(dir, "http://localhost:8080") + if err != nil { + t.Fatal(err) + } + + if l := len(index.Entries); l != 2 { + t.Fatalf("Expected 2 entries, got %d", l) + } + + // Other things test the entry generation more thoroughly. We just test a + // few fields. + cname := "frobnitz-1.2.3" + frob, ok := index.Entries[cname] + if !ok { + t.Fatalf("Could not read chart %s", cname) + } + if len(frob.Digest) == 0 { + t.Errorf("Missing digest of file %s.", frob.Name) + } + if frob.Chartfile == nil { + t.Fatalf("Chartfile %s not added to index.", cname) + } + if frob.URL != "http://localhost:8080/frobnitz-1.2.3.tgz" { + t.Errorf("Unexpected URL: %s", frob.URL) + } + if frob.Chartfile.Name != "frobnitz" { + t.Errorf("Expected frobnitz, got %q", frob.Chartfile.Name) + } +}