Support piping input to plugin

Signed-off-by: Dave Henderson <dhenderson@gmail.com>

8 files changed, 247 insertions, 37 deletions

diff --git a/docs/content/config.md b/docs/content/config.md
index 22b6f239..91b82fc6 100644
--- a/docs/content/config.md
+++ b/docs/content/config.md
@@ -300,22 +300,72 @@ outputMap: |
 
 See [`--plugin`](../usage/#--plugin).
 
-An mapping of key/value pairs to plug in custom functions for use in the templates.
+A map that configures custom functions for use in the templates. The key is the
+name of the function, and the value configures the plugin. The value is a map
+containing the command (`cmd`) and the options `pipe` (boolean) and `timeout`
+(duration).
+
+Alternatively, the value can be a string, which sets `cmd`.
 
 ```yaml
 in: '{{ "hello world" | figlet | lolcat }}'
 plugins:
-  figlet: /usr/local/bin/figlet
+  figlet:
+    cmd: /usr/local/bin/figlet
+    pipe: true
+    timeout: 1s
   lolcat: /home/hairyhenderson/go/bin/lolcat
 ```
 
+### `cmd`
+
+The path to the plugin executable (or script) to run.
+
+### `pipe`
+
+Whether to pipe the final argument of the template function to the plugin's
+Stdin, or provide as a separate argument.
+
+For example, given a `myfunc` plugin with a `cmd` of `/bin/myfunc`:
+
+With this template:
+```
+{{ print "bar" | myfunc "foo" }}
+```
+
+If `pipe` is `true`, the plugin executable will receive the input `"bar"` as its
+Stdin, like this shell command:
+
+```console
+$ echo -n "bar" | /bin/myfunc "foo"
+```
+
+If `pipe` is `false` (the default), the plugin executable will receive the
+input `"bar"` as its last argument, like this shell command:
+
+```console
+$ /bin/myfunc "foo" "bar"
+```
+
+_Note:_ in a chained pipeline (e.g. `{{ foo | bar }}`), the result of each
+command is passed as the final argument of the next, and so the template above
+could be written as `{{ myfunc "foo" "bar" }}`.
+
+### `timeout`
+
+The plugin's timeout. After this time, the command will be terminated and the
+template function will return an error. The value must be a valid
+[duration][] such as `1s`, `1m`, `1h`,
+
+The default is `5s`.
+
 ## `pluginTimeout`
 
 See [`--plugin`](../usage/#--plugin).
 
-Sets the timeout for running plugins. By default, plugins will time out after 5
-seconds. This value can be set to override this default. The value must be
-a valid [duration](../functions/time/#time-parseduration) such as `10s` or `3m`.
+Sets the timeout for all configured plugins. Overrides the default of `5s`.
+After this time, plugin commands will be killed. The value must be a valid
+[duration][] such as `10s` or `3m`.
 
 ```yaml
 plugins:
@@ -368,3 +418,4 @@ templates:
 [command-line arguments]: ../usage
 [file an issue]: https://github.com/hairyhenderson/gomplate/issues/new
 [YAML]: http://yaml.org
+[duration]: (../functions/time/#time-parseduration)
diff --git a/docs/content/usage.md b/docs/content/usage.md
index c231955f..b4af259e 100644
--- a/docs/content/usage.md
+++ b/docs/content/usage.md
@@ -236,6 +236,8 @@ A few different forms are valid:
 
 ### `--plugin`
 
+_See the [config file](../config/#plugins) for more plugin configuration options._
+
 Some specialized use cases may need functionality that gomplate isn't capable
 of on its own. If you have a command or script to perform this functionality,
 you can plug in your own custom functions with the `--plugin` flag:
@@ -357,7 +359,7 @@ post-exec command.
 
 ## Suppressing empty output
 
-Sometimes it can be desirable to suppress empty output (i.e. output consisting of only whitespace). To do so, set `suppressEmpty: true` in your [config][] file, or `GOMPLATE_SUPPRESS_EMPTY=true` in your environment:
+Sometimes it can be desirable to suppress empty output (i.e. output consisting of only whitespace). To do so, set `suppressEmpty: true` in your [config](../config/#suppressempty) file, or `GOMPLATE_SUPPRESS_EMPTY=true` in your environment:
 
 ```console
 $ export GOMPLATE_SUPPRESS_EMPTY=true
@@ -368,6 +370,5 @@ cat: out: No such file or directory
 
 [default context]: ../syntax/#the-context
 [context]: ../syntax/#the-context
-[config]: ../config/#suppressempty
 [external templates]: ../syntax/#external-templates
 [`.gitignore`]: https://git-scm.com/docs/gitignore
diff --git a/internal/config/configfile.go b/internal/config/configfile.go
index 3858f85e..459f9f9e 100644
--- a/internal/config/configfile.go
+++ b/internal/config/configfile.go
@@ -54,9 +54,9 @@ type Config struct {
 
 	PostExec []string `yaml:"postExec,omitempty,flow"`
 
-	DataSources map[string]DataSource `yaml:"datasources,omitempty"`
-	Context     map[string]DataSource `yaml:"context,omitempty"`
-	Plugins     map[string]string     `yaml:"plugins,omitempty"`
+	DataSources map[string]DataSource   `yaml:"datasources,omitempty"`
+	Context     map[string]DataSource   `yaml:"context,omitempty"`
+	Plugins     map[string]PluginConfig `yaml:"plugins,omitempty"`
 
 	// Extra HTTP headers not attached to pre-defined datsources. Potentially
 	// used by datasources defined in the template.
@@ -163,6 +163,47 @@ func (d DataSource) mergeFrom(o DataSource) DataSource {
 	return d
 }
 
+type PluginConfig struct {
+	Cmd     string
+	Timeout time.Duration
+	Pipe    bool
+}
+
+// UnmarshalYAML - satisfy the yaml.Umarshaler interface - plugin configs can
+// either be a plain string (to specify only the name), or a map with a name,
+// timeout, and pipe flag.
+func (p *PluginConfig) UnmarshalYAML(value *yaml.Node) error {
+	if value.Kind == yaml.ScalarNode {
+		s := ""
+		err := value.Decode(&s)
+		if err != nil {
+			return err
+		}
+
+		*p = PluginConfig{Cmd: s}
+		return nil
+	}
+
+	if value.Kind != yaml.MappingNode {
+		return fmt.Errorf("plugin config must be a string or map")
+	}
+
+	type raw struct {
+		Cmd     string
+		Timeout time.Duration
+		Pipe    bool
+	}
+	r := raw{}
+	err := value.Decode(&r)
+	if err != nil {
+		return err
+	}
+
+	*p = PluginConfig(r)
+
+	return nil
+}
+
 // MergeFrom - use this Config as the defaults, and override it with any
 // non-zero values from the other Config
 //
@@ -290,9 +331,9 @@ func (c *Config) ParsePluginFlags(plugins []string) error {
 			return fmt.Errorf("plugin requires both name and path")
 		}
 		if c.Plugins == nil {
-			c.Plugins = map[string]string{}
+			c.Plugins = map[string]PluginConfig{}
 		}
-		c.Plugins[parts[0]] = parts[1]
+		c.Plugins[parts[0]] = PluginConfig{Cmd: parts[1]}
 	}
 	return nil
 }
diff --git a/internal/config/configfile_test.go b/internal/config/configfile_test.go
index 2c65467a..e22ddf94 100644
--- a/internal/config/configfile_test.go
+++ b/internal/config/configfile_test.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/hairyhenderson/gomplate/v3/internal/iohelpers"
 	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v3"
 )
 
 func TestParseConfigFile(t *testing.T) {
@@ -42,6 +43,11 @@ context:
   .:
     url: file:///data.json
 
+plugins:
+  foo:
+    cmd: echo
+    pipe: true
+
 pluginTimeout: 2s
 `
 	expected = &Config{
@@ -63,7 +69,10 @@ pluginTimeout: 2s
 				URL: mustURL("file:///data.json"),
 			},
 		},
-		OutMode:       "644",
+		OutMode: "644",
+		Plugins: map[string]PluginConfig{
+			"foo": {Cmd: "echo", Pipe: true},
+		},
 		PluginTimeout: 2 * time.Second,
 	}
 
@@ -298,23 +307,23 @@ func TestMergeFrom(t *testing.T) {
 	cfg = &Config{
 		Input:       "hello world",
 		OutputFiles: []string{"-"},
-		Plugins: map[string]string{
-			"sleep": "echo",
+		Plugins: map[string]PluginConfig{
+			"sleep": {Cmd: "echo"},
 		},
 		PluginTimeout: 500 * time.Microsecond,
 	}
 	other = &Config{
 		InputFiles:  []string{"-"},
 		OutputFiles: []string{"-"},
-		Plugins: map[string]string{
-			"sleep": "sleep.sh",
+		Plugins: map[string]PluginConfig{
+			"sleep": {Cmd: "sleep.sh"},
 		},
 	}
 	expected = &Config{
 		Input:       "hello world",
 		OutputFiles: []string{"-"},
-		Plugins: map[string]string{
-			"sleep": "sleep.sh",
+		Plugins: map[string]PluginConfig{
+			"sleep": {Cmd: "sleep.sh"},
 		},
 		PluginTimeout: 500 * time.Microsecond,
 	}
@@ -394,7 +403,7 @@ func TestParsePluginFlags(t *testing.T) {
 	cfg = &Config{}
 	err = cfg.ParsePluginFlags([]string{"foo=bar"})
 	assert.NoError(t, err)
-	assert.EqualValues(t, &Config{Plugins: map[string]string{"foo": "bar"}}, cfg)
+	assert.EqualValues(t, &Config{Plugins: map[string]PluginConfig{"foo": {Cmd: "bar"}}}, cfg)
 }
 
 func TestConfigString(t *testing.T) {
@@ -476,6 +485,24 @@ pluginTimeout: 500ms
 `
 
 	assert.Equal(t, expected, c.String())
+
+	c = &Config{
+		Plugins: map[string]PluginConfig{
+			"foo": {
+				Cmd:  "bar",
+				Pipe: true,
+			},
+		},
+	}
+	expected = `---
+plugins:
+  foo:
+    cmd: bar
+    timeout: 0s
+    pipe: true
+`
+
+	assert.Equal(t, expected, c.String())
 }
 
 func TestApplyDefaults(t *testing.T) {
@@ -756,3 +783,35 @@ func TestFromContext(t *testing.T) {
 	// assert that the returned config looks like a default one
 	assert.Equal(t, "{{", cfg.LDelim)
 }
+
+func TestPluginConfig_UnmarshalYAML(t *testing.T) {
+	in := `foo`
+	out := PluginConfig{}
+	err := yaml.Unmarshal([]byte(in), &out)
+	assert.NoError(t, err)
+	assert.EqualValues(t, PluginConfig{Cmd: "foo"}, out)
+
+	in = `[foo, bar]`
+	out = PluginConfig{}
+	err = yaml.Unmarshal([]byte(in), &out)
+	assert.Error(t, err)
+
+	in = `cmd: foo`
+	out = PluginConfig{}
+	err = yaml.Unmarshal([]byte(in), &out)
+	assert.NoError(t, err)
+	assert.EqualValues(t, PluginConfig{Cmd: "foo"}, out)
+
+	in = `cmd: foo
+timeout: 10ms
+pipe: true
+`
+	out = PluginConfig{}
+	err = yaml.Unmarshal([]byte(in), &out)
+	assert.NoError(t, err)
+	assert.EqualValues(t, PluginConfig{
+		Cmd:     "foo",
+		Timeout: time.Duration(10) * time.Millisecond,
+		Pipe:    true,
+	}, out)
+}
diff --git a/internal/tests/integration/config_test.go b/internal/tests/integration/config_test.go
index 4ae47a10..7f31bfb6 100644
--- a/internal/tests/integration/config_test.go
+++ b/internal/tests/integration/config_test.go
@@ -28,8 +28,11 @@ func writeFile(dir *fs.Dir, f, content string) {
 	}
 }
 
-func writeConfig(dir *fs.Dir, content string) {
+func writeConfig(t *testing.T, dir *fs.Dir, content string) {
+	t.Helper()
+
 	writeFile(dir, ".gomplate.yaml", content)
+	t.Logf("writing config: %s", content)
 }
 
 func TestConfig_ReadsFromSimpleConfigFile(t *testing.T) {
@@ -41,7 +44,7 @@ func TestConfig_ReadsFromSimpleConfigFile(t *testing.T) {
 
 func TestConfig_ReadsStdin(t *testing.T) {
 	tmpDir := setupConfigTest(t)
-	writeConfig(tmpDir, "inputFiles: [-]")
+	writeConfig(t, tmpDir, "inputFiles: [-]")
 
 	o, e, err := cmd(t).withDir(tmpDir.Path()).withStdin("foo bar").run()
 	assertSuccess(t, o, e, err, "foo bar")
@@ -49,7 +52,7 @@ func TestConfig_ReadsStdin(t *testing.T) {
 
 func TestConfig_FlagOverridesConfig(t *testing.T) {
 	tmpDir := setupConfigTest(t)
-	writeConfig(tmpDir, "inputFiles: [in]")
+	writeConfig(t, tmpDir, "inputFiles: [in]")
 
 	o, e, err := cmd(t, "-i", "hello from the cli").
 		withDir(tmpDir.Path()).run()
@@ -58,7 +61,7 @@ func TestConfig_FlagOverridesConfig(t *testing.T) {
 
 func TestConfig_ReadsFromInputFile(t *testing.T) {
 	tmpDir := setupConfigTest(t)
-	writeConfig(tmpDir, "inputFiles: [in]")
+	writeConfig(t, tmpDir, "inputFiles: [in]")
 	writeFile(tmpDir, "in", "blah blah")
 
 	o, e, err := cmd(t).withDir(tmpDir.Path()).run()
@@ -67,7 +70,7 @@ func TestConfig_ReadsFromInputFile(t *testing.T) {
 
 func TestConfig_Datasource(t *testing.T) {
 	tmpDir := setupConfigTest(t)
-	writeConfig(tmpDir, `inputFiles: [in]
+	writeConfig(t, tmpDir, `inputFiles: [in]
 datasources:
   data:
     url: in.yaml
@@ -82,7 +85,7 @@ datasources:
 func TestConfig_OutputDir(t *testing.T) {
 	tmpDir := setupConfigTest(t)
 
-	writeConfig(tmpDir, `inputDir: indir/
+	writeConfig(t, tmpDir, `inputDir: indir/
 outputDir: outdir/
 datasources:
   data:
@@ -103,7 +106,7 @@ func TestConfig_ExecPipeOverridesConfigFile(t *testing.T) {
 	tmpDir := setupConfigTest(t)
 
 	// make sure exec-pipe works, and outFiles is replaced
-	writeConfig(tmpDir, `in: hello world
+	writeConfig(t, tmpDir, `in: hello world
 outputFiles: ['-']
 `)
 	o, e, err := cmd(t, "-i", "hi", "--exec-pipe", "--", "tr", "[a-z]", "[A-Z]").
@@ -114,7 +117,7 @@ outputFiles: ['-']
 func TestConfig_OutFile(t *testing.T) {
 	tmpDir := setupConfigTest(t)
 
-	writeConfig(tmpDir, `in: hello world
+	writeConfig(t, tmpDir, `in: hello world
 outputFiles: [out]
 `)
 	o, e, err := cmd(t).withDir(tmpDir.Path()).run()
@@ -152,7 +155,7 @@ func TestConfig_ConfigOverridesEnvDelim(t *testing.T) {
 
 	tmpDir := setupConfigTest(t)
 
-	writeConfig(tmpDir, `inputFiles: [in]
+	writeConfig(t, tmpDir, `inputFiles: [in]
 leftDelim: (╯°□°）╯︵ ┻━┻
 datasources:
   data:
@@ -175,7 +178,7 @@ func TestConfig_FlagOverridesAllDelim(t *testing.T) {
 
 	tmpDir := setupConfigTest(t)
 
-	writeConfig(tmpDir, `inputFiles: [in]
+	writeConfig(t, tmpDir, `inputFiles: [in]
 leftDelim: (╯°□°）╯︵ ┻━┻
 datasources:
   data:
@@ -199,7 +202,7 @@ func TestConfig_ConfigOverridesEnvPluginTimeout(t *testing.T) {
 
 	tmpDir := setupConfigTest(t)
 
-	writeConfig(tmpDir, `in: hi there {{ sleep 2 }}
+	writeConfig(t, tmpDir, `in: hi there {{ sleep 2 }}
 plugins:
   sleep: echo
 
@@ -215,7 +218,7 @@ pluginTimeout: 500ms
 func TestConfig_ConfigOverridesEnvSuppressEmpty(t *testing.T) {
 	tmpDir := setupConfigTest(t)
 
-	writeConfig(tmpDir, `in: |
+	writeConfig(t, tmpDir, `in: |
   {{- print "\t  \n\n\r\n\t\t     \v\n" -}}
 
   {{ print "   " -}}
diff --git a/internal/tests/integration/plugins_test.go b/internal/tests/integration/plugins_test.go
index c95b024d..451274d8 100644
--- a/internal/tests/integration/plugins_test.go
+++ b/internal/tests/integration/plugins_test.go
@@ -27,6 +27,13 @@ write-error $msg
 exit $code
 `, fs.WithMode(0755)),
 		fs.WithFile("sleep.sh", "#!/bin/sh\n\nexec sleep $1\n", fs.WithMode(0755)),
+		fs.WithFile("replace.sh", `#!/bin/sh
+if [ "$#" -eq 2 ]; then
+	exec tr $1 $2
+elif [ "$#" -eq 3 ]; then
+	printf "=%s" $3 | tr $1 $2
+fi
+`, fs.WithMode(0755)),
 	)
 	t.Cleanup(tmpDir.Remove)
 
@@ -58,6 +65,10 @@ func TestPlugins_Errors(t *testing.T) {
 }
 
 func TestPlugins_Timeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip()
+	}
+
 	tmpDir := setupPluginsTest(t)
 	_, _, err := cmd(t, "--plugin", "sleep="+tmpDir.Join("sleep.sh"),
 		"-i", `{{ sleep 10 }}`).run()
@@ -68,3 +79,30 @@ func TestPlugins_Timeout(t *testing.T) {
 		withEnv("GOMPLATE_PLUGIN_TIMEOUT", "500ms").run()
 	assert.ErrorContains(t, err, "plugin timed out")
 }
+
+func TestPlugins_PipeMode(t *testing.T) {
+	tmpDir := setupPluginsTest(t)
+
+	writeConfig(t, tmpDir, `in: '{{ "hi there" | replace "h" "H" }}'
+plugins:
+  replace:
+    cmd: `+tmpDir.Join("replace.sh")+`
+    pipe: true
+`)
+
+	o, e, err := cmd(t).withDir(tmpDir.Path()).run()
+	assert.NilError(t, err)
+	assert.Equal(t, "", e)
+	assert.Equal(t, "Hi tHere", o)
+
+	writeConfig(t, tmpDir, `in: '{{ "hi there" | replace "e" "Z" }}'
+plugins:
+  replace:
+    cmd: `+tmpDir.Join("replace.sh")+`
+`)
+
+	o, e, err = cmd(t).withDir(tmpDir.Path()).run()
+	assert.NilError(t, err)
+	assert.Equal(t, "", e)
+	assert.Equal(t, "=hi=thZrZ", o)
+}
diff --git a/plugins.go b/plugins.go
index 89309c0e..3e6e1f04 100644
--- a/plugins.go
+++ b/plugins.go
@@ -19,11 +19,17 @@ import (
 
 func bindPlugins(ctx context.Context, cfg *config.Config, funcMap template.FuncMap) error {
 	for k, v := range cfg.Plugins {
+		timeout := cfg.PluginTimeout
+		if v.Timeout != 0 {
+			timeout = v.Timeout
+		}
+
 		plugin := &plugin{
 			ctx:     ctx,
 			name:    k,
-			path:    v,
-			timeout: cfg.PluginTimeout,
+			path:    v.Cmd,
+			timeout: timeout,
+			pipe:    v.Pipe,
 			stderr:  cfg.Stderr,
 		}
 		if _, ok := funcMap[plugin.name]; ok {
@@ -40,6 +46,7 @@ type plugin struct {
 	stderr     io.Writer
 	name, path string
 	timeout    time.Duration
+	pipe       bool
 }
 
 // builds a command that's appropriate for running scripts
@@ -79,8 +86,18 @@ func (p *plugin) run(args ...interface{}) (interface{}, error) {
 
 	ctx, cancel := context.WithTimeout(p.ctx, p.timeout)
 	defer cancel()
+
+	var stdin *bytes.Buffer
+	if p.pipe && len(a) > 0 {
+		stdin = bytes.NewBufferString(a[len(a)-1])
+		a = a[:len(a)-1]
+	}
+
 	c := exec.CommandContext(ctx, name, a...)
-	c.Stdin = nil
+	if stdin != nil {
+		c.Stdin = stdin
+	}
+
 	c.Stderr = p.stderr
 	outBuf := &bytes.Buffer{}
 	c.Stdout = outBuf
diff --git a/plugins_test.go b/plugins_test.go
index 8590ed34..1e00a908 100644
--- a/plugins_test.go
+++ b/plugins_test.go
@@ -18,13 +18,13 @@ func TestBindPlugins(t *testing.T) {
 	ctx := context.Background()
 	fm := template.FuncMap{}
 	cfg := &config.Config{
-		Plugins: map[string]string{},
+		Plugins: map[string]config.PluginConfig{},
 	}
 	err := bindPlugins(ctx, cfg, fm)
 	assert.NilError(t, err)
 	assert.DeepEqual(t, template.FuncMap{}, fm)
 
-	cfg.Plugins = map[string]string{"foo": "bar"}
+	cfg.Plugins = map[string]config.PluginConfig{"foo": {Cmd: "bar"}}
 	err = bindPlugins(ctx, cfg, fm)
 	assert.NilError(t, err)
 	assert.Check(t, cmp.Contains(fm, "foo"))