Add --input-dir and --output-dir as options (#119)

All filese from --input-dir will be processed as templates and stored
with the same directory hierachy in --ouput-dir

- Use both options when a whole directory hierarchy needs to be processed.
- Extracted file processing logic in an extra process.go
- --output-dir is optional and default to "."
- --output-dir is created automatically if not existing

Fixes #117

Signed-off-by: Roland Huss <roland@ro14nd.de>

9 files changed, 318 insertions, 67 deletions

diff --git a/README.md b/README.md
index 9c0da279..9ca740bd 100644
--- a/README.md
+++ b/README.md
@@ -194,6 +194,18 @@ By default, `gomplate` will read from `Stdin` and write to `Stdout`. This behavi
 
 You can specify multiple `--file` and `--out` arguments. The same number of each much be given. This allows `gomplate` to process multiple templates _slightly_ faster than invoking `gomplate` multiple times in a row.
 
+##### `--input-dir` and `--output-dir`
+
+For processing multiple templates in a directory you can use `--input-dir` and `--output-dir` together. In this case all files in input directory will be processed as templates and the resulting files stored in `--output-dir`. The output directory will be created if it does not exist and the directory structure of the input directory will be preserved.  
+
+Example:
+
+```bash
+# Process all files in directory "templates" with the datasource given
+# and store the files with the same directory structure in "config"
+gomplate --input-dir=templates --output-dir=config --datasource config=config.yaml
+```
+
 #### `--datasource`/`-d`
 
 Add a data source in `name=URL` form. Specify multiple times to add multiple sources. The data can then be used by the [`datasource`](#datasource) function.
diff --git a/main.go b/main.go
index 66840296..808a7f72 100644
--- a/main.go
+++ b/main.go
@@ -2,13 +2,14 @@ package main
 
 import (
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 
 	"strings"
 	"text/template"
 
+	"errors"
+
 	"github.com/hairyhenderson/gomplate/aws"
 	"github.com/hairyhenderson/gomplate/version"
 	"github.com/urfave/cli"
@@ -32,7 +33,6 @@ func (g *Gomplate) RunTemplate(text string, out io.Writer) {
 	if err != nil {
 		log.Fatalf("Line %q: %v\n", text, err)
 	}
-
 	if err := tmpl.Execute(out, context); err != nil {
 		panic(err)
 	}
@@ -78,43 +78,6 @@ func NewGomplate(data *Data, leftDelim, rightDelim string) *Gomplate {
 	}
 }
 
-func readInputs(input string, files []string) []string {
-	if input != "" {
-		return []string{input}
-	}
-	if len(files) == 0 {
-		files = []string{"-"}
-	}
-	ins := make([]string, len(files))
-
-	for n, filename := range files {
-		var err error
-		var inFile *os.File
-		if filename == "-" {
-			inFile = os.Stdin
-		} else {
-			inFile, err = os.Open(filename)
-			if err != nil {
-				log.Fatalf("Failed to open %s\n%v", filename, err)
-			}
-			defer inFile.Close() // nolint: errcheck
-		}
-		bytes, err := ioutil.ReadAll(inFile)
-		if err != nil {
-			log.Fatalf("Read failed for %s!\n%v\n", filename, err)
-		}
-		ins[n] = string(bytes)
-	}
-	return ins
-}
-
-func openOutFile(filename string) (out *os.File, err error) {
-	if filename == "-" {
-		return os.Stdout, nil
-	}
-	return os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
-}
-
 func runTemplate(c *cli.Context) error {
 	defer runCleanupHooks()
 	data := NewData(c.StringSlice("datasource"), c.StringSlice("datasource-header"))
@@ -123,22 +86,51 @@ func runTemplate(c *cli.Context) error {
 
 	g := NewGomplate(data, lDelim, rDelim)
 
-	inputs := readInputs(c.String("in"), c.StringSlice("file"))
+	if err := validateInOutOptions(c); err != nil {
+		return err
+	}
 
-	outputs := c.StringSlice("out")
-	if len(outputs) == 0 {
-		outputs = []string{"-"}
+	inputDir := c.String("input-dir")
+	if inputDir != "" {
+		return processInputDir(inputDir, getOutputDir(c), g)
 	}
 
-	for n, input := range inputs {
-		out, err := openOutFile(outputs[n])
-		if err != nil {
-			return err
-		}
-		defer out.Close() // nolint: errcheck
-		g.RunTemplate(input, out)
+	return processInputFiles(c.String("in"), c.StringSlice("file"), c.StringSlice("out"), g)
+}
+func getOutputDir(c *cli.Context) string {
+	out := c.String("output-dir")
+	if out != "" {
+		return out
 	}
+	return "."
+}
 
+// Called from process.go ...
+func renderTemplate(g *Gomplate, inString string, outPath string) error {
+	outFile, err := openOutFile(outPath)
+	if err != nil {
+		return err
+	}
+	// nolint: errcheck
+	defer outFile.Close()
+	g.RunTemplate(inString, outFile)
+	return nil
+}
+
+func validateInOutOptions(c *cli.Context) error {
+	if c.String("input-dir") != "" {
+		if c.String("in") != "" || len(c.StringSlice("file")) != 0 {
+			return errors.New("--input-dir can not be used together with --in or --file")
+		}
+	}
+	if c.String("output-dir") != "" {
+		if len(c.StringSlice("out")) != 0 {
+			return errors.New("--out can not be used together with --output-dir")
+		}
+		if c.String("input-dir") == "" {
+			return errors.New("--input-dir must be set when --output-dir is set")
+		}
+	}
 	return nil
 }
 
@@ -152,16 +144,24 @@ func main() {
 	app.Flags = []cli.Flag{
 		cli.StringSliceFlag{
 			Name:  "file, f",
-			Usage: "Template file to process. Omit to use standard input (-), or use --in",
+			Usage: "Template file to process. Omit to use standard input (-), or use --in or --input-dir",
 		},
 		cli.StringFlag{
 			Name:  "in, i",
-			Usage: "Template string to process (alternative to --file)",
+			Usage: "Template string to process (alternative to --file and --input-dir)",
+		},
+		cli.StringFlag{
+			Name:  "input-dir",
+			Usage: "Directory which is examined recursively for templates (alternative to --file and --in)",
 		},
 		cli.StringSliceFlag{
 			Name:  "out, o",
 			Usage: "Output file name. Omit to use standard output (-).",
 		},
+		cli.StringFlag{
+			Name:  "output-dir",
+			Usage: "Directory to store the processed templates. Only used for --input-dir",
+		},
 		cli.StringSliceFlag{
 			Name:  "datasource, d",
 			Usage: "Data source in alias=URL form. Specify multiple times to add multiple sources.",
diff --git a/main_test.go b/main_test.go
index 268d2afb..e35e6660 100644
--- a/main_test.go
+++ b/main_test.go
@@ -2,7 +2,6 @@ package main
 
 import (
 	"bytes"
-	"io/ioutil"
 	"net/http/httptest"
 	"os"
 	"testing"
@@ -150,17 +149,3 @@ func TestCustomDelim(t *testing.T) {
 	}
 	assert.Equal(t, "hi", testTemplate(g, `[print "hi"]`))
 }
-
-func TestReadInput(t *testing.T) {
-	actual := readInputs("foo", nil)
-	assert.Equal(t, "foo", actual[0])
-
-	// stdin is "" because during tests it's given /dev/null
-	actual = readInputs("", []string{"-"})
-	assert.Equal(t, "", actual[0])
-
-	actual = readInputs("", []string{"main_test.go"})
-	thisFile, _ := os.Open("main_test.go")
-	expected, _ := ioutil.ReadAll(thisFile)
-	assert.Equal(t, string(expected), actual[0])
-}
diff --git a/process.go b/process.go
new file mode 100644
index 00000000..85235a52
--- /dev/null
+++ b/process.go
@@ -0,0 +1,123 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// == Direct input processing ========================================
+
+func processInputFiles(stringTemplate string, input []string, output []string, g *Gomplate) error {
+	input, err := readInputs(stringTemplate, input)
+	if err != nil {
+		return err
+	}
+
+	if len(output) == 0 {
+		output = []string{"-"}
+	}
+
+	for n, input := range input {
+		if err := renderTemplate(g, input, output[n]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// == Recursive input dir processing ======================================
+
+func processInputDir(input string, output string, g *Gomplate) error {
+	input = filepath.Clean(input)
+	output = filepath.Clean(output)
+
+	// assert tha input path exists
+	si, err := os.Stat(input)
+	if err != nil {
+		return err
+	}
+
+	// read directory
+	entries, err := ioutil.ReadDir(input)
+	if err != nil {
+		return err
+	}
+
+	// ensure output directory
+	if err = os.MkdirAll(output, si.Mode()); err != nil {
+		return err
+	}
+
+	// process or dive in again
+	for _, entry := range entries {
+		nextInPath := filepath.Join(input, entry.Name())
+		nextOutPath := filepath.Join(output, entry.Name())
+
+		if entry.IsDir() {
+			err := processInputDir(nextInPath, nextOutPath, g)
+			if err != nil {
+				return err
+			}
+		} else {
+			inString, err := readInput(nextInPath)
+			if err != nil {
+				return err
+			}
+			if err := renderTemplate(g, inString, nextOutPath); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// == File handling ================================================
+
+func readInputs(input string, files []string) ([]string, error) {
+	if input != "" {
+		return []string{input}, nil
+	}
+	if len(files) == 0 {
+		files = []string{"-"}
+	}
+	ins := make([]string, len(files))
+
+	for n, filename := range files {
+		inString, err := readInput(filename)
+		if err != nil {
+			return nil, err
+		}
+		ins[n] = inString
+	}
+	return ins, nil
+}
+
+func readInput(filename string) (string, error) {
+	var err error
+	var inFile *os.File
+	if filename == "-" {
+		inFile = os.Stdin
+	} else {
+		inFile, err = os.Open(filename)
+		if err != nil {
+			return "", fmt.Errorf("failed to open %s\n%v", filename, err)
+		}
+		// nolint: errcheck
+		defer inFile.Close()
+	}
+	bytes, err := ioutil.ReadAll(inFile)
+	if err != nil {
+		err = fmt.Errorf("read failed for %s\n%v", filename, err)
+		return "", err
+	}
+	return string(bytes), nil
+}
+
+func openOutFile(filename string) (out *os.File, err error) {
+	if filename == "-" {
+		return os.Stdout, nil
+	}
+	return os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+}
diff --git a/process_test.go b/process_test.go
new file mode 100644
index 00000000..5b7fc812
--- /dev/null
+++ b/process_test.go
@@ -0,0 +1,58 @@
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"path/filepath"
+
+	"log"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadInput(t *testing.T) {
+	actual, err := readInputs("foo", nil)
+	assert.Nil(t, err)
+	assert.Equal(t, "foo", actual[0])
+
+	// stdin is "" because during tests it's given /dev/null
+	actual, err = readInputs("", []string{"-"})
+	assert.Nil(t, err)
+	assert.Equal(t, "", actual[0])
+
+	actual, err = readInputs("", []string{"main_test.go"})
+	assert.Nil(t, err)
+	thisFile, _ := os.Open("main_test.go")
+	expected, _ := ioutil.ReadAll(thisFile)
+	assert.Equal(t, string(expected), actual[0])
+}
+
+func TestInputDir(t *testing.T) {
+	outDir, err := ioutil.TempDir("test/files/input-dir", "out-temp-")
+	assert.Nil(t, err)
+	defer (func() {
+		if cerr := os.RemoveAll(outDir); cerr != nil {
+			log.Fatalf("Error while removing temporary directory %s : %v", outDir, cerr)
+		}
+	})()
+
+	src, err := ParseSource("config=test/files/input-dir/config.yml")
+	assert.Nil(t, err)
+
+	data := &Data{
+		Sources: map[string]*Source{"config": src},
+	}
+	gomplate := NewGomplate(data, "{{", "}}")
+	err = processInputDir("test/files/input-dir/in", outDir, gomplate)
+	assert.Nil(t, err)
+
+	top, err := ioutil.ReadFile(filepath.Join(outDir, "top.txt"))
+	assert.Nil(t, err)
+	assert.Equal(t, "eins", string(top))
+
+	inner, err := ioutil.ReadFile(filepath.Join(outDir, "inner/nested.txt"))
+	assert.Nil(t, err)
+	assert.Equal(t, "zwei", string(inner))
+}
diff --git a/test/files/input-dir/config.yml b/test/files/input-dir/config.yml
new file mode 100644
index 00000000..1ec99548
--- /dev/null
+++ b/test/files/input-dir/config.yml
@@ -0,0 +1,2 @@
+one: eins
+two: zwei
diff --git a/test/files/input-dir/in/inner/nested.txt b/test/files/input-dir/in/inner/nested.txt
new file mode 100644
index 00000000..55e06b79
--- /dev/null
+++ b/test/files/input-dir/in/inner/nested.txt
@@ -0,0 +1 @@
+{{ (datasource "config").two }}
\ No newline at end of file
diff --git a/test/files/input-dir/in/top.txt b/test/files/input-dir/in/top.txt
new file mode 100644
index 00000000..9069510b
--- /dev/null
+++ b/test/files/input-dir/in/top.txt
@@ -0,0 +1 @@
+{{ (datasource "config").one }}
\ No newline at end of file
diff --git a/test/integration/input-dir.bats b/test/integration/input-dir.bats
new file mode 100644
index 00000000..72667a29
--- /dev/null
+++ b/test/integration/input-dir.bats
@@ -0,0 +1,69 @@
+#!/usr/bin/env bats
+
+load helper
+
+tmpdir=$(mktemp -u)
+
+function setup () {
+  mkdir -p $tmpdir
+  mkdir -p $tmpdir/in/inner
+  echo -n "{{ (datasource \"config\").one }}" > $tmpdir/in/eins.txt
+  echo -n "{{ (datasource \"config\").two }}" > $tmpdir/in/inner/deux.txt
+
+  cat <<"EOT"  > $tmpdir/config.yml
+one: eins
+two: deux
+EOT
+}
+
+function teardown () {
+  # rm -rf $tmpdir
+  echo
+}
+
+@test "takes --input-dir and produces proper output files" {
+  rm -rf $tmpdir/out || true
+  gomplate --input-dir $tmpdir/in --output-dir $tmpdir/out -d config=$tmpdir/config.yml
+  [ "$status" -eq 0 ]
+  [[ "$(ls $tmpdir/out | wc -l)" == 2 ]]
+  [[ "$(ls $tmpdir/out/inner | wc -l)" == 1 ]]
+  [[ "$(cat $tmpdir/out/eins.txt)" == "eins" ]]
+  [[ "$(cat $tmpdir/out/inner/deux.txt)" == "deux" ]]
+}
+
+@test "test . as default --output-dir param" {
+  rm -rf $tmpdir/out_dot || true
+  mkdir -p $tmpdir/out_dot
+  g=$(pwd)/bin/gomplate
+  cd $tmpdir/out_dot
+  run $g --input-dir $tmpdir/in -d config=$tmpdir/config.yml
+  [ "$?" -eq 0 ]
+  [[ "$(ls | wc -l)" == 2 ]]
+  [[ "$(ls inner | wc -l)" == 1 ]]
+  [[ "$(cat eins.txt)" == "eins" ]]
+  [[ "$(cat inner/deux.txt)" == "deux" ]]
+}
+
+@test "errors given --output-dir but no --input-dir" {
+  gomplate --output-dir "."
+  [ "$status" -eq 1 ]
+  [[ "${output}" == "--input-dir must be set when --output-dir is set" ]]
+}
+
+@test "errors given both --input-dir and --in" {
+  gomplate --input-dir "." --in "param"
+  [ "$status" -eq 1 ]
+  [[ "${output}" == "--input-dir can not be used together with --in or --file" ]]
+}
+
+@test "errors given both --input-dir and --file" {
+  gomplate --input-dir "." --file input.txt
+  [ "$status" -eq 1 ]
+  [[ "${output}" == "--input-dir can not be used together with --in or --file" ]]
+}
+
+@test "errors given both --output-dir and --out" {
+  gomplate --input-dir "." --output-dir /tmp --out out
+  [ "$status" -eq 1 ]
+  [[ "${output}" == "--out can not be used together with --output-dir" ]]
+}