From f1d9158ea99abbe556251c1ff2fe970f3b460ee9 Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Thu, 25 Jan 2024 20:04:03 -0500 Subject: Remove support for deprecated key/value array form of template config (#1976) Signed-off-by: Dave Henderson --- docs/content/config.md | 12 ----- internal/config/configfile.go | 7 +-- internal/config/configfile_test.go | 14 +++--- internal/config/types.go | 78 ------------------------------- internal/config/types_test.go | 29 ++---------- internal/tests/integration/config_test.go | 14 ------ render.go | 47 ++++--------------- render_test.go | 9 ++-- template.go | 4 +- template_test.go | 2 +- 10 files changed, 30 insertions(+), 186 deletions(-) diff --git a/docs/content/config.md b/docs/content/config.md index dd947b99..60eac24a 100644 --- a/docs/content/config.md +++ b/docs/content/config.md @@ -455,18 +455,6 @@ templates: url: https://example.com/api/v1/someremotetemplate header: Authorization: ["Basic aGF4MHI6c3dvcmRmaXNoCg=="] - dir: foo/bar/ -``` - -_(Deprecated)_ Can also be an array of template references. Can be just a path, -or an alias and a path: - -```yaml -templates: - - t=foo/bar/helloworld.tmpl - - templatedir/ - - dir=foo/bar/ - - mytemplate.t ``` [command-line arguments]: ../usage diff --git a/internal/config/configfile.go b/internal/config/configfile.go index 1fcccc20..f22d9a1b 100644 --- a/internal/config/configfile.go +++ b/internal/config/configfile.go @@ -40,7 +40,7 @@ type Config struct { DataSources map[string]DataSource `yaml:"datasources,omitempty"` Context map[string]DataSource `yaml:"context,omitempty"` Plugins map[string]PluginConfig `yaml:"plugins,omitempty"` - Templates Templates `yaml:"templates,omitempty"` + Templates map[string]DataSource `yaml:"templates,omitempty"` // Extra HTTP headers not attached to pre-defined datsources. Potentially // used by datasources defined in the template. @@ -107,7 +107,7 @@ type DataSource struct { // well supported, and anyway we need to do some extra parsing func (d *DataSource) UnmarshalYAML(value *yaml.Node) error { type raw struct { - Header http.Header + Header http.Header `yaml:",omitempty"` URL string } r := raw{} @@ -130,9 +130,10 @@ func (d *DataSource) UnmarshalYAML(value *yaml.Node) error { // well supported, and anyway we need to do some extra parsing func (d DataSource) MarshalYAML() (interface{}, error) { type raw struct { - Header http.Header + Header http.Header `yaml:",omitempty"` URL string } + r := raw{ URL: d.URL.String(), Header: d.Header, diff --git a/internal/config/configfile_test.go b/internal/config/configfile_test.go index 449c379b..d2759a1e 100644 --- a/internal/config/configfile_test.go +++ b/internal/config/configfile_test.go @@ -74,7 +74,7 @@ pluginTimeout: 2s Plugins: map[string]PluginConfig{ "foo": {Cmd: "echo", Pipe: true}, }, - Templates: Templates{"foo": DataSource{URL: mustURL("file:///tmp/foo.t")}}, + Templates: map[string]DataSource{"foo": {URL: mustURL("file:///tmp/foo.t")}}, PluginTimeout: 2 * time.Second, } @@ -386,7 +386,7 @@ func TestMergeFrom(t *testing.T) { cfg = &Config{ InputDir: "indir/", ExcludeGlob: []string{"*.txt"}, - Templates: Templates{ + Templates: map[string]DataSource{ "foo": { URL: mustURL("file:///foo.yaml"), }, @@ -402,7 +402,7 @@ func TestMergeFrom(t *testing.T) { OutMode: "600", LDelim: "${", RDelim: "}", - Templates: Templates{ + Templates: map[string]DataSource{ "foo": {URL: mustURL("https://example.com/foo.yaml")}, "baz": {URL: mustURL("vault:///baz")}, }, @@ -414,7 +414,7 @@ func TestMergeFrom(t *testing.T) { OutMode: "600", LDelim: "${", RDelim: "}", - Templates: Templates{ + Templates: map[string]DataSource{ "foo": {URL: mustURL("https://example.com/foo.yaml")}, "bar": { URL: mustURL("stdin:///"), @@ -503,7 +503,7 @@ func TestParseDataSourceFlags(t *testing.T) { ) require.NoError(t, err) assert.EqualValues(t, &Config{ - Templates: Templates{ + Templates: map[string]DataSource{ "foo": { URL: mustURL("http://example.com"), Header: http.Header{"Accept": {"application/json"}}, @@ -551,7 +551,7 @@ pluginTimeout: 5s RDelim: "R", Input: "foo", OutputFiles: []string{"-"}, - Templates: Templates{ + Templates: map[string]DataSource{ "foo": {URL: mustURL("https://www.example.com/foo.tmpl")}, "bar": {URL: mustURL("file:///tmp/bar.t")}, }, @@ -576,7 +576,7 @@ templates: RDelim: "R", Input: "long input that should be truncated", OutputFiles: []string{"-"}, - Templates: Templates{ + Templates: map[string]DataSource{ "foo": {URL: mustURL("https://www.example.com/foo.tmpl")}, "bar": {URL: mustURL("file:///tmp/bar.t")}, }, diff --git a/internal/config/types.go b/internal/config/types.go index 648ad2b9..25414105 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -1,89 +1,11 @@ package config import ( - "fmt" - "net/http" "strings" "github.com/hairyhenderson/gomplate/v4/internal/urlhelpers" - "github.com/hairyhenderson/yaml" ) -// Templates - a map of templates. We can't just use map[string]DataSource, -// because we need to be able to marshal both the old (array of '[k=]v' strings) -// and the new (proper map) formats. -// -// Note that templates use the DataSource type, since they have the exact same -// shape. -// TODO: get rid of this and just use map[string]DataSource once the legacy -// [k=]v array format is no longer supported -type Templates map[string]DataSource - -// UnmarshalYAML - satisfy the yaml.Umarshaler interface -func (t *Templates) UnmarshalYAML(value *yaml.Node) error { - // first attempt to unmarshal as a map[string]DataSource - m := map[string]DataSource{} - err := value.Decode(m) - if err == nil { - *t = m - return nil - } - - // if that fails, try to unmarshal as an array of '[k=]v' strings - err = t.unmarshalYAMLArray(value) - if err != nil { - return fmt.Errorf("could not unmarshal templates as map or array: %w", err) - } - - return nil -} - -func (t *Templates) unmarshalYAMLArray(value *yaml.Node) error { - a := []string{} - err := value.Decode(&a) - if err != nil { - return fmt.Errorf("could not unmarshal templates as array: %w", err) - } - - ts := Templates{} - for _, s := range a { - alias, pth, _ := strings.Cut(s, "=") - if pth == "" { - // when alias is omitted, the path and alias are identical - pth = alias - } - - u, err := urlhelpers.ParseSourceURL(pth) - if err != nil { - return fmt.Errorf("could not parse template URL %q: %w", pth, err) - } - - ts[alias] = DataSource{ - URL: u, - } - } - - *t = ts - - return nil -} - -func (t Templates) MarshalYAML() (interface{}, error) { - type rawTemplate struct { - Header http.Header `yaml:"header,omitempty,flow"` - URL string `yaml:"url"` - } - - m := map[string]rawTemplate{} - for k, v := range t { - m[k] = rawTemplate{ - Header: v.Header, - URL: v.URL.String(), - } - } - return m, nil -} - func parseTemplateArg(value string) (alias string, ds DataSource, err error) { alias, u, _ := strings.Cut(value, "=") if u == "" { diff --git a/internal/config/types_test.go b/internal/config/types_test.go index 47942c2f..f4f66fc9 100644 --- a/internal/config/types_test.go +++ b/internal/config/types_test.go @@ -22,10 +22,10 @@ remote: url: https://example.com/foo/bar/helloworld.tmpl header: Accept: [text/plain, text/template]` - out := Templates{} + out := map[string]DataSource{} err := yaml.Unmarshal([]byte(in), &out) require.NoError(t, err) - assert.EqualValues(t, Templates{ + assert.EqualValues(t, map[string]DataSource{ "t": {URL: mustURL("foo/bar/helloworld.tmpl")}, "templatedir": {URL: mustURL("templatedir/")}, "dir": {URL: mustURL("foo/bar/")}, @@ -36,32 +36,9 @@ remote: }, }, out) - // legacy array format - in = `- t=foo/bar/helloworld.tmpl -- templatedir/ -- dir=foo/bar/ -- mytemplate.t -- remote=https://example.com/foo/bar/helloworld.tmpl` - out = Templates{} - err = yaml.Unmarshal([]byte(in), &out) - require.NoError(t, err) - assert.EqualValues(t, Templates{ - "t": {URL: mustURL("foo/bar/helloworld.tmpl")}, - "templatedir/": {URL: mustURL("templatedir/")}, - "dir": {URL: mustURL("foo/bar/")}, - "mytemplate.t": {URL: mustURL("mytemplate.t")}, - "remote": {URL: mustURL("https://example.com/foo/bar/helloworld.tmpl")}, - }, out) - // invalid format in = `"neither an array nor a map"` - out = Templates{} - err = yaml.Unmarshal([]byte(in), &out) - assert.Error(t, err) - - // invalid URL - in = `- t="not a:valid url"` - out = Templates{} + out = map[string]DataSource{} err = yaml.Unmarshal([]byte(in), &out) assert.Error(t, err) } diff --git a/internal/tests/integration/config_test.go b/internal/tests/integration/config_test.go index 00ab4f89..5c414bff 100644 --- a/internal/tests/integration/config_test.go +++ b/internal/tests/integration/config_test.go @@ -262,20 +262,6 @@ templates: assertSuccess(t, o, e, err, "12345") } -func TestConfig_ConfigTemplatesSupportsArray(t *testing.T) { - tmpDir := setupConfigTest(t) - - // TODO: remove this test once the array format is no longer supported - writeConfig(t, tmpDir, `in: '{{ template "t1" (dict "testValue" "12345") }}' -templates: - - t1=t1.tmpl -`) - writeFile(t, tmpDir, "t1.tmpl", `{{ .testValue }}`) - - o, e, err := cmd(t).withDir(tmpDir.Path()).run() - assertSuccess(t, o, e, err, "12345") -} - func TestConfig_MissingKeyDefault(t *testing.T) { tmpDir := setupConfigTest(t) writeConfig(t, tmpDir, `inputFiles: [in] diff --git a/render.go b/render.go index 7b377291..9ea8c1e1 100644 --- a/render.go +++ b/render.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "net/http" - "net/url" "os" "sync" "text/template" @@ -28,12 +27,12 @@ type Options struct { // Datasources - map of datasources to be read on demand when the // 'datasource'/'ds'/'include' functions are used. - Datasources map[string]Datasource + Datasources map[string]config.DataSource // Context - map of datasources to be read immediately and added to the // template's context - Context map[string]Datasource + Context map[string]config.DataSource // Templates - map of templates that can be referenced as nested templates - Templates map[string]Datasource + Templates map[string]config.DataSource // Extra HTTP headers not attached to pre-defined datsources. Potentially // used by datasources defined in the template. @@ -60,32 +59,10 @@ type Options struct { // optionsFromConfig - create a set of options from the internal config struct. // Does not set the Funcs field. func optionsFromConfig(cfg *config.Config) Options { - ds := make(map[string]Datasource, len(cfg.DataSources)) - for k, v := range cfg.DataSources { - ds[k] = Datasource{ - URL: v.URL, - Header: v.Header, - } - } - cs := make(map[string]Datasource, len(cfg.Context)) - for k, v := range cfg.Context { - cs[k] = Datasource{ - URL: v.URL, - Header: v.Header, - } - } - ts := make(map[string]Datasource, len(cfg.Templates)) - for k, v := range cfg.Templates { - ts[k] = Datasource{ - URL: v.URL, - Header: v.Header, - } - } - opts := Options{ - Datasources: ds, - Context: cs, - Templates: ts, + Datasources: cfg.DataSources, + Context: cfg.Context, + Templates: cfg.Templates, ExtraHeaders: cfg.ExtraHeaders, LDelim: cfg.LDelim, RDelim: cfg.RDelim, @@ -96,14 +73,6 @@ func optionsFromConfig(cfg *config.Config) Options { return opts } -// Datasource - a datasource URL with optional headers -// -// Experimental: subject to breaking changes before the next major release -type Datasource struct { - URL *url.URL - Header http.Header -} - // Renderer provides gomplate's core template rendering functionality. // It should be initialized with NewRenderer. // @@ -112,7 +81,7 @@ type Renderer struct { //nolint:staticcheck data *data.Data fsp fsimpl.FSProvider - nested config.Templates + nested map[string]config.DataSource funcs template.FuncMap lDelim string rDelim string @@ -149,7 +118,7 @@ func NewRenderer(opts Options) *Renderer { // convert the internal config.Templates to a map[string]Datasource // TODO: simplify when config.Templates is removed - nested := config.Templates{} + nested := map[string]config.DataSource{} for alias, ds := range opts.Templates { nested[alias] = config.DataSource{ URL: ds.URL, diff --git a/render_test.go b/render_test.go index ab6f58f2..f7ed38ff 100644 --- a/render_test.go +++ b/render_test.go @@ -11,6 +11,7 @@ import ( "testing/fstest" "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/gomplate/v4/internal/config" "github.com/hairyhenderson/gomplate/v4/internal/datafs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -44,10 +45,10 @@ func TestRenderTemplate(t *testing.T) { t.Setenv("WORLD", "world") tr = NewRenderer(Options{ - Context: map[string]Datasource{ + Context: map[string]config.DataSource{ "hi": {URL: hu}, }, - Datasources: map[string]Datasource{ + Datasources: map[string]config.DataSource{ "world": {URL: wu}, }, }) @@ -63,7 +64,7 @@ func TestRenderTemplate(t *testing.T) { `<< . | toUpper >>`)} tr = NewRenderer(Options{ - Templates: map[string]Datasource{ + Templates: map[string]config.DataSource{ "nested": {URL: nu}, }, LDelim: "<<", @@ -146,7 +147,7 @@ func ExampleRenderer_datasources() { // a datasource that retrieves JSON from a public API u, _ := url.Parse("https://ipinfo.io/1.1.1.1") tr := NewRenderer(Options{ - Context: map[string]Datasource{ + Context: map[string]config.DataSource{ "info": {URL: u}, }, }) diff --git a/template.go b/template.go index 8c0aa64a..2705e7de 100644 --- a/template.go +++ b/template.go @@ -49,7 +49,7 @@ func copyFuncMap(funcMap template.FuncMap) template.FuncMap { } // parseTemplate - parses text as a Go template with the given name and options -func parseTemplate(ctx context.Context, name, text string, funcs template.FuncMap, tmplctx interface{}, nested config.Templates, leftDelim, rightDelim string, missingKey string) (tmpl *template.Template, err error) { +func parseTemplate(ctx context.Context, name, text string, funcs template.FuncMap, tmplctx interface{}, nested map[string]config.DataSource, leftDelim, rightDelim string, missingKey string) (tmpl *template.Template, err error) { tmpl = template.New(name) if missingKey == "" { missingKey = "error" @@ -81,7 +81,7 @@ func parseTemplate(ctx context.Context, name, text string, funcs template.FuncMa return tmpl, nil } -func parseNestedTemplates(ctx context.Context, nested config.Templates, tmpl *template.Template) error { +func parseNestedTemplates(ctx context.Context, nested map[string]config.DataSource, tmpl *template.Template) error { fsp := datafs.FSProviderFromContext(ctx) for alias, n := range nested { diff --git a/template_test.go b/template_test.go index d21d693e..3a0757a4 100644 --- a/template_test.go +++ b/template_test.go @@ -184,7 +184,7 @@ func TestParseNestedTemplates(t *testing.T) { // simple test with single template u, _ := url.Parse("foo.t") - nested := config.Templates{"foo": {URL: u}} + nested := map[string]config.DataSource{"foo": {URL: u}} tmpl, _ := template.New("root").Parse(`{{ template "foo" }}`) -- cgit v1.2.3