From 2cc314740e4ce29739c667f0887448d6ee592542 Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Thu, 25 Jan 2024 21:14:06 -0500 Subject: Revert "Remove support for deprecated key/value array form of template config" (#1979) --- docs/content/config.md | 12 +++++ internal/config/configfile.go | 7 ++- internal/config/configfile_test.go | 14 +++--- internal/config/types.go | 78 +++++++++++++++++++++++++++++++ internal/config/types_test.go | 29 ++++++++++-- internal/tests/integration/config_test.go | 14 ++++++ render.go | 47 +++++++++++++++---- render_test.go | 9 ++-- template.go | 4 +- template_test.go | 2 +- 10 files changed, 186 insertions(+), 30 deletions(-) diff --git a/docs/content/config.md b/docs/content/config.md index 60eac24a..dd947b99 100644 --- a/docs/content/config.md +++ b/docs/content/config.md @@ -455,6 +455,18 @@ templates: url: https://example.com/api/v1/someremotetemplate header: Authorization: ["Basic aGF4MHI6c3dvcmRmaXNoCg=="] + dir: foo/bar/ +``` + +_(Deprecated)_ Can also be an array of template references. Can be just a path, +or an alias and a path: + +```yaml +templates: + - t=foo/bar/helloworld.tmpl + - templatedir/ + - dir=foo/bar/ + - mytemplate.t ``` [command-line arguments]: ../usage diff --git a/internal/config/configfile.go b/internal/config/configfile.go index f22d9a1b..1fcccc20 100644 --- a/internal/config/configfile.go +++ b/internal/config/configfile.go @@ -40,7 +40,7 @@ type Config struct { DataSources map[string]DataSource `yaml:"datasources,omitempty"` Context map[string]DataSource `yaml:"context,omitempty"` Plugins map[string]PluginConfig `yaml:"plugins,omitempty"` - Templates map[string]DataSource `yaml:"templates,omitempty"` + Templates Templates `yaml:"templates,omitempty"` // Extra HTTP headers not attached to pre-defined datsources. Potentially // used by datasources defined in the template. @@ -107,7 +107,7 @@ type DataSource struct { // well supported, and anyway we need to do some extra parsing func (d *DataSource) UnmarshalYAML(value *yaml.Node) error { type raw struct { - Header http.Header `yaml:",omitempty"` + Header http.Header URL string } r := raw{} @@ -130,10 +130,9 @@ func (d *DataSource) UnmarshalYAML(value *yaml.Node) error { // well supported, and anyway we need to do some extra parsing func (d DataSource) MarshalYAML() (interface{}, error) { type raw struct { - Header http.Header `yaml:",omitempty"` + Header http.Header URL string } - r := raw{ URL: d.URL.String(), Header: d.Header, diff --git a/internal/config/configfile_test.go b/internal/config/configfile_test.go index d2759a1e..449c379b 100644 --- a/internal/config/configfile_test.go +++ b/internal/config/configfile_test.go @@ -74,7 +74,7 @@ pluginTimeout: 2s Plugins: map[string]PluginConfig{ "foo": {Cmd: "echo", Pipe: true}, }, - Templates: map[string]DataSource{"foo": {URL: mustURL("file:///tmp/foo.t")}}, + Templates: Templates{"foo": DataSource{URL: mustURL("file:///tmp/foo.t")}}, PluginTimeout: 2 * time.Second, } @@ -386,7 +386,7 @@ func TestMergeFrom(t *testing.T) { cfg = &Config{ InputDir: "indir/", ExcludeGlob: []string{"*.txt"}, - Templates: map[string]DataSource{ + Templates: Templates{ "foo": { URL: mustURL("file:///foo.yaml"), }, @@ -402,7 +402,7 @@ func TestMergeFrom(t *testing.T) { OutMode: "600", LDelim: "${", RDelim: "}", - Templates: map[string]DataSource{ + Templates: Templates{ "foo": {URL: mustURL("https://example.com/foo.yaml")}, "baz": {URL: mustURL("vault:///baz")}, }, @@ -414,7 +414,7 @@ func TestMergeFrom(t *testing.T) { OutMode: "600", LDelim: "${", RDelim: "}", - Templates: map[string]DataSource{ + Templates: Templates{ "foo": {URL: mustURL("https://example.com/foo.yaml")}, "bar": { URL: mustURL("stdin:///"), @@ -503,7 +503,7 @@ func TestParseDataSourceFlags(t *testing.T) { ) require.NoError(t, err) assert.EqualValues(t, &Config{ - Templates: map[string]DataSource{ + Templates: Templates{ "foo": { URL: mustURL("http://example.com"), Header: http.Header{"Accept": {"application/json"}}, @@ -551,7 +551,7 @@ pluginTimeout: 5s RDelim: "R", Input: "foo", OutputFiles: []string{"-"}, - Templates: map[string]DataSource{ + Templates: Templates{ "foo": {URL: mustURL("https://www.example.com/foo.tmpl")}, "bar": {URL: mustURL("file:///tmp/bar.t")}, }, @@ -576,7 +576,7 @@ templates: RDelim: "R", Input: "long input that should be truncated", OutputFiles: []string{"-"}, - Templates: map[string]DataSource{ + Templates: Templates{ "foo": {URL: mustURL("https://www.example.com/foo.tmpl")}, "bar": {URL: mustURL("file:///tmp/bar.t")}, }, diff --git a/internal/config/types.go b/internal/config/types.go index 25414105..648ad2b9 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -1,11 +1,89 @@ package config import ( + "fmt" + "net/http" "strings" "github.com/hairyhenderson/gomplate/v4/internal/urlhelpers" + "github.com/hairyhenderson/yaml" ) +// Templates - a map of templates. We can't just use map[string]DataSource, +// because we need to be able to marshal both the old (array of '[k=]v' strings) +// and the new (proper map) formats. +// +// Note that templates use the DataSource type, since they have the exact same +// shape. +// TODO: get rid of this and just use map[string]DataSource once the legacy +// [k=]v array format is no longer supported +type Templates map[string]DataSource + +// UnmarshalYAML - satisfy the yaml.Umarshaler interface +func (t *Templates) UnmarshalYAML(value *yaml.Node) error { + // first attempt to unmarshal as a map[string]DataSource + m := map[string]DataSource{} + err := value.Decode(m) + if err == nil { + *t = m + return nil + } + + // if that fails, try to unmarshal as an array of '[k=]v' strings + err = t.unmarshalYAMLArray(value) + if err != nil { + return fmt.Errorf("could not unmarshal templates as map or array: %w", err) + } + + return nil +} + +func (t *Templates) unmarshalYAMLArray(value *yaml.Node) error { + a := []string{} + err := value.Decode(&a) + if err != nil { + return fmt.Errorf("could not unmarshal templates as array: %w", err) + } + + ts := Templates{} + for _, s := range a { + alias, pth, _ := strings.Cut(s, "=") + if pth == "" { + // when alias is omitted, the path and alias are identical + pth = alias + } + + u, err := urlhelpers.ParseSourceURL(pth) + if err != nil { + return fmt.Errorf("could not parse template URL %q: %w", pth, err) + } + + ts[alias] = DataSource{ + URL: u, + } + } + + *t = ts + + return nil +} + +func (t Templates) MarshalYAML() (interface{}, error) { + type rawTemplate struct { + Header http.Header `yaml:"header,omitempty,flow"` + URL string `yaml:"url"` + } + + m := map[string]rawTemplate{} + for k, v := range t { + m[k] = rawTemplate{ + Header: v.Header, + URL: v.URL.String(), + } + } + return m, nil +} + func parseTemplateArg(value string) (alias string, ds DataSource, err error) { alias, u, _ := strings.Cut(value, "=") if u == "" { diff --git a/internal/config/types_test.go b/internal/config/types_test.go index f4f66fc9..47942c2f 100644 --- a/internal/config/types_test.go +++ b/internal/config/types_test.go @@ -22,10 +22,10 @@ remote: url: https://example.com/foo/bar/helloworld.tmpl header: Accept: [text/plain, text/template]` - out := map[string]DataSource{} + out := Templates{} err := yaml.Unmarshal([]byte(in), &out) require.NoError(t, err) - assert.EqualValues(t, map[string]DataSource{ + assert.EqualValues(t, Templates{ "t": {URL: mustURL("foo/bar/helloworld.tmpl")}, "templatedir": {URL: mustURL("templatedir/")}, "dir": {URL: mustURL("foo/bar/")}, @@ -36,9 +36,32 @@ remote: }, }, out) + // legacy array format + in = `- t=foo/bar/helloworld.tmpl +- templatedir/ +- dir=foo/bar/ +- mytemplate.t +- remote=https://example.com/foo/bar/helloworld.tmpl` + out = Templates{} + err = yaml.Unmarshal([]byte(in), &out) + require.NoError(t, err) + assert.EqualValues(t, Templates{ + "t": {URL: mustURL("foo/bar/helloworld.tmpl")}, + "templatedir/": {URL: mustURL("templatedir/")}, + "dir": {URL: mustURL("foo/bar/")}, + "mytemplate.t": {URL: mustURL("mytemplate.t")}, + "remote": {URL: mustURL("https://example.com/foo/bar/helloworld.tmpl")}, + }, out) + // invalid format in = `"neither an array nor a map"` - out = map[string]DataSource{} + out = Templates{} + err = yaml.Unmarshal([]byte(in), &out) + assert.Error(t, err) + + // invalid URL + in = `- t="not a:valid url"` + out = Templates{} err = yaml.Unmarshal([]byte(in), &out) assert.Error(t, err) } diff --git a/internal/tests/integration/config_test.go b/internal/tests/integration/config_test.go index 5c414bff..00ab4f89 100644 --- a/internal/tests/integration/config_test.go +++ b/internal/tests/integration/config_test.go @@ -262,6 +262,20 @@ templates: assertSuccess(t, o, e, err, "12345") } +func TestConfig_ConfigTemplatesSupportsArray(t *testing.T) { + tmpDir := setupConfigTest(t) + + // TODO: remove this test once the array format is no longer supported + writeConfig(t, tmpDir, `in: '{{ template "t1" (dict "testValue" "12345") }}' +templates: + - t1=t1.tmpl +`) + writeFile(t, tmpDir, "t1.tmpl", `{{ .testValue }}`) + + o, e, err := cmd(t).withDir(tmpDir.Path()).run() + assertSuccess(t, o, e, err, "12345") +} + func TestConfig_MissingKeyDefault(t *testing.T) { tmpDir := setupConfigTest(t) writeConfig(t, tmpDir, `inputFiles: [in] diff --git a/render.go b/render.go index 9ea8c1e1..7b377291 100644 --- a/render.go +++ b/render.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "sync" "text/template" @@ -27,12 +28,12 @@ type Options struct { // Datasources - map of datasources to be read on demand when the // 'datasource'/'ds'/'include' functions are used. - Datasources map[string]config.DataSource + Datasources map[string]Datasource // Context - map of datasources to be read immediately and added to the // template's context - Context map[string]config.DataSource + Context map[string]Datasource // Templates - map of templates that can be referenced as nested templates - Templates map[string]config.DataSource + Templates map[string]Datasource // Extra HTTP headers not attached to pre-defined datsources. Potentially // used by datasources defined in the template. @@ -59,10 +60,32 @@ type Options struct { // optionsFromConfig - create a set of options from the internal config struct. // Does not set the Funcs field. func optionsFromConfig(cfg *config.Config) Options { + ds := make(map[string]Datasource, len(cfg.DataSources)) + for k, v := range cfg.DataSources { + ds[k] = Datasource{ + URL: v.URL, + Header: v.Header, + } + } + cs := make(map[string]Datasource, len(cfg.Context)) + for k, v := range cfg.Context { + cs[k] = Datasource{ + URL: v.URL, + Header: v.Header, + } + } + ts := make(map[string]Datasource, len(cfg.Templates)) + for k, v := range cfg.Templates { + ts[k] = Datasource{ + URL: v.URL, + Header: v.Header, + } + } + opts := Options{ - Datasources: cfg.DataSources, - Context: cfg.Context, - Templates: cfg.Templates, + Datasources: ds, + Context: cs, + Templates: ts, ExtraHeaders: cfg.ExtraHeaders, LDelim: cfg.LDelim, RDelim: cfg.RDelim, @@ -73,6 +96,14 @@ func optionsFromConfig(cfg *config.Config) Options { return opts } +// Datasource - a datasource URL with optional headers +// +// Experimental: subject to breaking changes before the next major release +type Datasource struct { + URL *url.URL + Header http.Header +} + // Renderer provides gomplate's core template rendering functionality. // It should be initialized with NewRenderer. // @@ -81,7 +112,7 @@ type Renderer struct { //nolint:staticcheck data *data.Data fsp fsimpl.FSProvider - nested map[string]config.DataSource + nested config.Templates funcs template.FuncMap lDelim string rDelim string @@ -118,7 +149,7 @@ func NewRenderer(opts Options) *Renderer { // convert the internal config.Templates to a map[string]Datasource // TODO: simplify when config.Templates is removed - nested := map[string]config.DataSource{} + nested := config.Templates{} for alias, ds := range opts.Templates { nested[alias] = config.DataSource{ URL: ds.URL, diff --git a/render_test.go b/render_test.go index f7ed38ff..ab6f58f2 100644 --- a/render_test.go +++ b/render_test.go @@ -11,7 +11,6 @@ import ( "testing/fstest" "github.com/hairyhenderson/go-fsimpl" - "github.com/hairyhenderson/gomplate/v4/internal/config" "github.com/hairyhenderson/gomplate/v4/internal/datafs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -45,10 +44,10 @@ func TestRenderTemplate(t *testing.T) { t.Setenv("WORLD", "world") tr = NewRenderer(Options{ - Context: map[string]config.DataSource{ + Context: map[string]Datasource{ "hi": {URL: hu}, }, - Datasources: map[string]config.DataSource{ + Datasources: map[string]Datasource{ "world": {URL: wu}, }, }) @@ -64,7 +63,7 @@ func TestRenderTemplate(t *testing.T) { `<< . | toUpper >>`)} tr = NewRenderer(Options{ - Templates: map[string]config.DataSource{ + Templates: map[string]Datasource{ "nested": {URL: nu}, }, LDelim: "<<", @@ -147,7 +146,7 @@ func ExampleRenderer_datasources() { // a datasource that retrieves JSON from a public API u, _ := url.Parse("https://ipinfo.io/1.1.1.1") tr := NewRenderer(Options{ - Context: map[string]config.DataSource{ + Context: map[string]Datasource{ "info": {URL: u}, }, }) diff --git a/template.go b/template.go index 2705e7de..8c0aa64a 100644 --- a/template.go +++ b/template.go @@ -49,7 +49,7 @@ func copyFuncMap(funcMap template.FuncMap) template.FuncMap { } // parseTemplate - parses text as a Go template with the given name and options -func parseTemplate(ctx context.Context, name, text string, funcs template.FuncMap, tmplctx interface{}, nested map[string]config.DataSource, leftDelim, rightDelim string, missingKey string) (tmpl *template.Template, err error) { +func parseTemplate(ctx context.Context, name, text string, funcs template.FuncMap, tmplctx interface{}, nested config.Templates, leftDelim, rightDelim string, missingKey string) (tmpl *template.Template, err error) { tmpl = template.New(name) if missingKey == "" { missingKey = "error" @@ -81,7 +81,7 @@ func parseTemplate(ctx context.Context, name, text string, funcs template.FuncMa return tmpl, nil } -func parseNestedTemplates(ctx context.Context, nested map[string]config.DataSource, tmpl *template.Template) error { +func parseNestedTemplates(ctx context.Context, nested config.Templates, tmpl *template.Template) error { fsp := datafs.FSProviderFromContext(ctx) for alias, n := range nested { diff --git a/template_test.go b/template_test.go index 3a0757a4..d21d693e 100644 --- a/template_test.go +++ b/template_test.go @@ -184,7 +184,7 @@ func TestParseNestedTemplates(t *testing.T) { // simple test with single template u, _ := url.Parse("foo.t") - nested := map[string]config.DataSource{"foo": {URL: u}} + nested := config.Templates{"foo": {URL: u}} tmpl, _ := template.New("root").Parse(`{{ template "foo" }}`) -- cgit v1.2.3