Only open output files when necessary

Signed-off-by: Dave Henderson <dhenderson@gmail.com>

12 files changed, 363 insertions, 29 deletions

diff --git a/go.mod b/go.mod
index 7db66783..6f9dfcc4 100644
--- a/go.mod
+++ b/go.mod
@@ -27,6 +27,7 @@ require (
 	github.com/zealic/xignore v0.3.3
 	gocloud.dev v0.20.0
 	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de
+	golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f
 	gopkg.in/src-d/go-billy.v4 v4.3.2
 	gopkg.in/src-d/go-git.v4 v4.13.1
diff --git a/gomplate_test.go b/gomplate_test.go
index 0142aae0..1d569958 100644
--- a/gomplate_test.go
+++ b/gomplate_test.go
@@ -16,7 +16,7 @@ import (
 	"github.com/hairyhenderson/gomplate/v3/conv"
 	"github.com/hairyhenderson/gomplate/v3/data"
 	"github.com/hairyhenderson/gomplate/v3/env"
-	"github.com/hairyhenderson/gomplate/v3/internal/writers"
+	"github.com/hairyhenderson/gomplate/v3/internal/iohelpers"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -158,7 +158,7 @@ func TestCustomDelim(t *testing.T) {
 func TestRunTemplates(t *testing.T) {
 	defer func() { Stdout = os.Stdout }()
 	buf := &bytes.Buffer{}
-	Stdout = &writers.NopCloser{Writer: buf}
+	Stdout = &iohelpers.NopCloser{Writer: buf}
 	config := &Config{Input: "foo", OutputFiles: []string{"-"}}
 	err := RunTemplates(config)
 	assert.NoError(t, err)
diff --git a/internal/iohelpers/readers.go b/internal/iohelpers/readers.go
new file mode 100644
index 00000000..277d7ab9
--- /dev/null
+++ b/internal/iohelpers/readers.go
@@ -0,0 +1,48 @@
+package iohelpers
+
+import (
+	"io"
+	"sync"
+)
+
+// LazyReadCloser provides an interface to a ReadCloser that will open on the
+// first access. The wrapped io.ReadCloser must be provided by 'open'.
+func LazyReadCloser(open func() (io.ReadCloser, error)) io.ReadCloser {
+	return &lazyReadCloser{
+		opened: sync.Once{},
+		open:   open,
+	}
+}
+
+type lazyReadCloser struct {
+	opened sync.Once
+	r      io.ReadCloser
+	// caches the error that came from open(), if any
+	openErr error
+	open    func() (io.ReadCloser, error)
+}
+
+var _ io.ReadCloser = (*lazyReadCloser)(nil)
+
+func (l *lazyReadCloser) openReader() (r io.ReadCloser, err error) {
+	l.opened.Do(func() {
+		l.r, l.openErr = l.open()
+	})
+	return l.r, l.openErr
+}
+
+func (l *lazyReadCloser) Close() error {
+	r, err := l.openReader()
+	if err != nil {
+		return err
+	}
+	return r.Close()
+}
+
+func (l *lazyReadCloser) Read(p []byte) (n int, err error) {
+	r, err := l.openReader()
+	if err != nil {
+		return 0, err
+	}
+	return r.Read(p)
+}
diff --git a/internal/iohelpers/readers_test.go b/internal/iohelpers/readers_test.go
new file mode 100644
index 00000000..ef300ef8
--- /dev/null
+++ b/internal/iohelpers/readers_test.go
@@ -0,0 +1,49 @@
+package iohelpers
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLazyReadCloser(t *testing.T) {
+	r := newBufferCloser(bytes.NewBufferString("hello world"))
+	opened := false
+	l, ok := LazyReadCloser(func() (io.ReadCloser, error) {
+		opened = true
+		return r, nil
+	}).(*lazyReadCloser)
+	assert.True(t, ok)
+
+	assert.False(t, opened)
+	assert.Nil(t, l.r)
+	assert.False(t, r.closed)
+
+	p := make([]byte, 5)
+	n, err := l.Read(p)
+	assert.NoError(t, err)
+	assert.True(t, opened)
+	assert.Equal(t, r, l.r)
+	assert.Equal(t, 5, n)
+
+	err = l.Close()
+	assert.NoError(t, err)
+	assert.True(t, r.closed)
+
+	// test error propagation
+	l = LazyReadCloser(func() (io.ReadCloser, error) {
+		return nil, os.ErrNotExist
+	}).(*lazyReadCloser)
+
+	assert.Nil(t, l.r)
+
+	p = make([]byte, 5)
+	_, err = l.Read(p)
+	assert.Error(t, err)
+
+	err = l.Close()
+	assert.Error(t, err)
+}
diff --git a/internal/writers/writers.go b/internal/iohelpers/writers.go
index b40f4e73..0358bc43 100644
--- a/internal/writers/writers.go
+++ b/internal/iohelpers/writers.go
@@ -1,4 +1,4 @@
-package writers
+package iohelpers
 
 import (
 	"bufio"
@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"sync"
 )
 
 type emptySkipper struct {
@@ -169,3 +170,45 @@ func (f *sameSkipper) Close() error {
 	}
 	return nil
 }
+
+// LazyWriteCloser provides an interface to a WriteCloser that will open on the
+// first access. The wrapped io.WriteCloser must be provided by 'open'.
+func LazyWriteCloser(open func() (io.WriteCloser, error)) io.WriteCloser {
+	return &lazyWriteCloser{
+		opened: sync.Once{},
+		open:   open,
+	}
+}
+
+type lazyWriteCloser struct {
+	opened sync.Once
+	w      io.WriteCloser
+	// caches the error that came from open(), if any
+	openErr error
+	open    func() (io.WriteCloser, error)
+}
+
+var _ io.WriteCloser = (*lazyWriteCloser)(nil)
+
+func (l *lazyWriteCloser) openWriter() (r io.WriteCloser, err error) {
+	l.opened.Do(func() {
+		l.w, l.openErr = l.open()
+	})
+	return l.w, l.openErr
+}
+
+func (l *lazyWriteCloser) Close() error {
+	w, err := l.openWriter()
+	if err != nil {
+		return err
+	}
+	return w.Close()
+}
+
+func (l *lazyWriteCloser) Write(p []byte) (n int, err error) {
+	w, err := l.openWriter()
+	if err != nil {
+		return 0, err
+	}
+	return w.Write(p)
+}
diff --git a/internal/writers/writers_test.go b/internal/iohelpers/writers_test.go
index b3452fcd..d226a03e 100644
--- a/internal/writers/writers_test.go
+++ b/internal/iohelpers/writers_test.go
@@ -1,9 +1,10 @@
-package writers
+package iohelpers
 
 import (
 	"bytes"
 	"fmt"
 	"io"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -37,7 +38,7 @@ func TestEmptySkipper(t *testing.T) {
 	}
 
 	for _, d := range testdata {
-		w := &bufferCloser{&bytes.Buffer{}}
+		w := newBufferCloser(&bytes.Buffer{})
 		opened := false
 		f, ok := NewEmptySkipper(func() (io.WriteCloser, error) {
 			opened = true
@@ -61,11 +62,18 @@ func TestEmptySkipper(t *testing.T) {
 	}
 }
 
+func newBufferCloser(b *bytes.Buffer) *bufferCloser {
+	return &bufferCloser{b, false}
+}
+
 type bufferCloser struct {
 	*bytes.Buffer
+
+	closed bool
 }
 
 func (b *bufferCloser) Close() error {
+	b.closed = true
 	return nil
 }
 
@@ -86,7 +94,7 @@ func TestSameSkipper(t *testing.T) {
 	for _, d := range testdata {
 		t.Run(fmt.Sprintf("in:%q/out:%q/same:%v", d.in, d.out, d.same), func(t *testing.T) {
 			r := bytes.NewBuffer(d.out)
-			w := &bufferCloser{&bytes.Buffer{}}
+			w := newBufferCloser(&bytes.Buffer{})
 			opened := false
 			f, ok := SameSkipper(r, func() (io.WriteCloser, error) {
 				opened = true
@@ -111,3 +119,41 @@ func TestSameSkipper(t *testing.T) {
 		})
 	}
 }
+
+func TestLazyWriteCloser(t *testing.T) {
+	w := newBufferCloser(&bytes.Buffer{})
+	opened := false
+	l, ok := LazyWriteCloser(func() (io.WriteCloser, error) {
+		opened = true
+		return w, nil
+	}).(*lazyWriteCloser)
+	assert.True(t, ok)
+
+	assert.False(t, opened)
+	assert.Nil(t, l.w)
+	assert.False(t, w.closed)
+
+	p := []byte("hello world")
+	n, err := l.Write(p)
+	assert.NoError(t, err)
+	assert.True(t, opened)
+	assert.Equal(t, 11, n)
+
+	err = l.Close()
+	assert.NoError(t, err)
+	assert.True(t, w.closed)
+
+	// test error propagation
+	l = LazyWriteCloser(func() (io.WriteCloser, error) {
+		return nil, os.ErrNotExist
+	}).(*lazyWriteCloser)
+
+	assert.Nil(t, l.w)
+
+	p = []byte("hello world")
+	_, err = l.Write(p)
+	assert.Error(t, err)
+
+	err = l.Close()
+	assert.Error(t, err)
+}
diff --git a/internal/tests/integration/inputdir_test.go b/internal/tests/integration/inputdir_test.go
index 3e3329da..ab791b48 100644
--- a/internal/tests/integration/inputdir_test.go
+++ b/internal/tests/integration/inputdir_test.go
@@ -43,10 +43,6 @@ out/{{ .in | strings.ReplaceAll $f (index .filemap $f) }}.out
 	)
 }
 
-func (s *InputDirSuite) TearDownTest(c *C) {
-	s.tmpDir.Remove()
-}
-
 func (s *InputDirSuite) TestInputDir(c *C) {
 	result := icmd.RunCommand(GomplateBin,
 		"--input-dir", s.tmpDir.Join("in"),
diff --git a/internal/tests/integration/inputdir_unix_test.go b/internal/tests/integration/inputdir_unix_test.go
new file mode 100644
index 00000000..5a6cde18
--- /dev/null
+++ b/internal/tests/integration/inputdir_unix_test.go
@@ -0,0 +1,70 @@
+//+build integration
+//+build !windows
+
+package integration
+
+import (
+	"fmt"
+	"io/ioutil"
+	"math"
+	"os"
+
+	. "gopkg.in/check.v1"
+
+	"golang.org/x/sys/unix"
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/fs"
+	"gotest.tools/v3/icmd"
+)
+
+func setFileUlimit(b uint64) error {
+	ulimit := unix.Rlimit{
+		Cur: b,
+		Max: math.MaxInt64,
+	}
+	err := unix.Setrlimit(unix.RLIMIT_NOFILE, &ulimit)
+	return err
+}
+
+func (s *InputDirSuite) TestInputDirRespectsUlimit(c *C) {
+	numfiles := 32
+	flist := map[string]string{}
+	for i := 0; i < numfiles; i++ {
+		k := fmt.Sprintf("file_%d", i)
+		flist[k] = fmt.Sprintf("hello world %d\n", i)
+	}
+	testdir := fs.NewDir(c, "ulimittestfiles",
+		fs.WithDir("in", fs.WithFiles(flist)),
+	)
+	defer testdir.Remove()
+
+	// we need another ~11 fds for other various things, so we'd be guaranteed
+	// to hit the limit if we try to have all the input files open
+	// simultaneously
+	setFileUlimit(uint64(numfiles))
+	defer setFileUlimit(8192)
+
+	result := icmd.RunCmd(icmd.Command(GomplateBin,
+		"--input-dir", testdir.Join("in"),
+		"--output-dir", testdir.Join("out"),
+	), func(c *icmd.Cmd) {
+		c.Dir = testdir.Path()
+	})
+	setFileUlimit(8192)
+	result.Assert(c, icmd.Success)
+
+	files, err := ioutil.ReadDir(testdir.Join("out"))
+	assert.NilError(c, err)
+	assert.Equal(c, numfiles, len(files))
+
+	for i := 0; i < numfiles; i++ {
+		f := testdir.Join("out", fmt.Sprintf("file_%d", i))
+		_, err := os.Stat(f)
+		assert.NilError(c, err)
+
+		content, err := ioutil.ReadFile(f)
+		assert.NilError(c, err)
+		expected := fmt.Sprintf("hello world %d\n", i)
+		assert.Equal(c, expected, string(content))
+	}
+}
diff --git a/template.go b/template.go
index b500ce73..448eff84 100644
--- a/template.go
+++ b/template.go
@@ -9,7 +9,7 @@ import (
 	"text/template"
 
 	"github.com/hairyhenderson/gomplate/v3/internal/config"
-	"github.com/hairyhenderson/gomplate/v3/internal/writers"
+	"github.com/hairyhenderson/gomplate/v3/internal/iohelpers"
 	"github.com/hairyhenderson/gomplate/v3/tmpl"
 
 	"github.com/spf13/afero"
@@ -101,7 +101,7 @@ func gatherTemplates(cfg *config.Config, outFileNamer func(string) (string, erro
 
 	// --exec-pipe redirects standard out to the out pipe
 	if cfg.OutWriter != nil {
-		Stdout = &writers.NopCloser{Writer: cfg.OutWriter}
+		Stdout = &iohelpers.NopCloser{Writer: cfg.OutWriter}
 	}
 
 	switch {
@@ -227,7 +227,7 @@ func fileToTemplates(inFile, outFile string, mode os.FileMode, modeOverride bool
 
 func openOutFile(cfg *config.Config, filename string, mode os.FileMode, modeOverride bool) (out io.WriteCloser, err error) {
 	if cfg.SuppressEmpty {
-		out = writers.NewEmptySkipper(func() (io.WriteCloser, error) {
+		out = iohelpers.NewEmptySkipper(func() (io.WriteCloser, error) {
 			if filename == "-" {
 				return Stdout, nil
 			}
@@ -260,14 +260,19 @@ func createOutFile(filename string, mode os.FileMode, modeOverride bool) (out io
 	}
 
 	// if the output file already exists, we'll use a SameSkipper
-	f, err := fs.OpenFile(filename, os.O_RDONLY, mode.Perm())
+	fi, err := fs.Stat(filename)
 	if err != nil {
-		// likely means the file just doesn't exist - open's error will be more useful
-		return open()
+		// likely means the file just doesn't exist - further errors will be more useful
+		return iohelpers.LazyWriteCloser(open), nil
 	}
-	out = writers.SameSkipper(f, func() (io.WriteCloser, error) {
-		return open()
-	})
+	if fi.IsDir() {
+		// error because this is a directory
+		return nil, isDirError(fi.Name())
+	}
+
+	out = iohelpers.SameSkipper(iohelpers.LazyReadCloser(func() (io.ReadCloser, error) {
+		return fs.OpenFile(filename, os.O_RDONLY, mode.Perm())
+	}), open)
 
 	return out, err
 }
@@ -280,14 +285,14 @@ func readInput(filename string) (string, error) {
 	} else {
 		inFile, err = fs.OpenFile(filename, os.O_RDONLY, 0)
 		if err != nil {
-			return "", fmt.Errorf("failed to open %s\n%v", filename, err)
+			return "", fmt.Errorf("failed to open %s: %w", filename, err)
 		}
 		// nolint: errcheck
 		defer inFile.Close()
 	}
 	bytes, err := ioutil.ReadAll(inFile)
 	if err != nil {
-		err = fmt.Errorf("read failed for %s\n%v", filename, err)
+		err = fmt.Errorf("read failed for %s: %w", filename, err)
 		return "", err
 	}
 	return string(bytes), nil
diff --git a/template_test.go b/template_test.go
index b0ba3698..0f77f4c0 100644
--- a/template_test.go
+++ b/template_test.go
@@ -8,10 +8,11 @@ import (
 	"testing"
 
 	"github.com/hairyhenderson/gomplate/v3/internal/config"
-	"github.com/hairyhenderson/gomplate/v3/internal/writers"
+	"github.com/hairyhenderson/gomplate/v3/internal/iohelpers"
 	"github.com/spf13/afero"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestReadInput(t *testing.T) {
@@ -47,16 +48,20 @@ func TestOpenOutFile(t *testing.T) {
 	_ = fs.Mkdir("/tmp", 0777)
 
 	cfg := &config.Config{}
-	_, err := openOutFile(cfg, "/tmp/foo", 0644, false)
+	f, err := openOutFile(cfg, "/tmp/foo", 0644, false)
 	assert.NoError(t, err)
+
+	err = f.Close()
+	assert.NoError(t, err)
+
 	i, err := fs.Stat("/tmp/foo")
 	assert.NoError(t, err)
 	assert.Equal(t, os.FileMode(0644), i.Mode())
 
 	defer func() { Stdout = os.Stdout }()
-	Stdout = &writers.NopCloser{Writer: &bytes.Buffer{}}
+	Stdout = &iohelpers.NopCloser{Writer: &bytes.Buffer{}}
 
-	f, err := openOutFile(cfg, "-", 0644, false)
+	f, err = openOutFile(cfg, "-", 0644, false)
 	assert.NoError(t, err)
 	assert.Equal(t, Stdout, f)
 }
@@ -120,8 +125,17 @@ func TestGatherTemplates(t *testing.T) {
 	assert.Len(t, templates, 1)
 	assert.Equal(t, "out", templates[0].targetPath)
 	assert.Equal(t, os.FileMode(0644), templates[0].mode)
-	info, err := fs.Stat("out")
+
+	// out file is created only on demand
+	_, err = fs.Stat("out")
+	assert.Error(t, err)
+	assert.True(t, os.IsNotExist(err))
+
+	_, err = templates[0].target.Write([]byte("hello world"))
 	assert.NoError(t, err)
+
+	info, err := fs.Stat("out")
+	require.NoError(t, err)
 	assert.Equal(t, os.FileMode(0644), info.Mode())
 	fs.Remove("out")
 
@@ -134,6 +148,10 @@ func TestGatherTemplates(t *testing.T) {
 	assert.Equal(t, "bar", templates[0].contents)
 	assert.NotEqual(t, Stdout, templates[0].target)
 	assert.Equal(t, os.FileMode(0600), templates[0].mode)
+
+	_, err = templates[0].target.Write([]byte("hello world"))
+	assert.NoError(t, err)
+
 	info, err = fs.Stat("out")
 	assert.NoError(t, err)
 	assert.Equal(t, os.FileMode(0600), info.Mode())
@@ -149,6 +167,10 @@ func TestGatherTemplates(t *testing.T) {
 	assert.Equal(t, "bar", templates[0].contents)
 	assert.NotEqual(t, Stdout, templates[0].target)
 	assert.Equal(t, os.FileMode(0755), templates[0].mode)
+
+	_, err = templates[0].target.Write([]byte("hello world"))
+	assert.NoError(t, err)
+
 	info, err = fs.Stat("out")
 	assert.NoError(t, err)
 	assert.Equal(t, os.FileMode(0755), info.Mode())
@@ -230,17 +252,26 @@ func TestProcessTemplates(t *testing.T) {
 		},
 	}
 	for _, in := range testdata {
+
 		actual, err := processTemplates(cfg, in.templates)
 		assert.NoError(t, err)
 		assert.Len(t, actual, len(in.templates))
 		for i, a := range actual {
+			current := in.templates[i]
 			assert.Equal(t, in.contents[i], a.contents)
-			assert.Equal(t, in.templates[i].mode, a.mode)
+			assert.Equal(t, current.mode, a.mode)
 			if len(in.targets) > 0 {
 				assert.Equal(t, in.targets[i], a.target)
 			}
-			if in.templates[i].targetPath != "-" {
-				info, err := fs.Stat(in.templates[i].targetPath)
+			if current.targetPath != "-" {
+				err = current.loadContents()
+				assert.NoError(t, err)
+
+				n, err := current.target.Write([]byte("hello world"))
+				assert.NoError(t, err)
+				assert.Equal(t, 11, n)
+
+				info, err := fs.Stat(current.targetPath)
 				assert.NoError(t, err)
 				assert.Equal(t, in.modes[i], info.Mode())
 			}
@@ -248,3 +279,14 @@ func TestProcessTemplates(t *testing.T) {
 		fs.Remove("out")
 	}
 }
+
+func TestCreateOutFile(t *testing.T) {
+	origfs := fs
+	defer func() { fs = origfs }()
+	fs = afero.NewMemMapFs()
+	_ = fs.Mkdir("in", 0755)
+
+	_, err := createOutFile("in", 0644, false)
+	assert.Error(t, err)
+	assert.IsType(t, &os.PathError{}, err)
+}
diff --git a/template_unix.go b/template_unix.go
new file mode 100644
index 00000000..950b7fa7
--- /dev/null
+++ b/template_unix.go
@@ -0,0 +1,17 @@
+//+build !windows
+
+package gomplate
+
+import (
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+func isDirError(name string) *os.PathError {
+	return &os.PathError{
+		Op:   "open",
+		Path: name,
+		Err:  unix.EISDIR,
+	}
+}
diff --git a/template_windows.go b/template_windows.go
new file mode 100644
index 00000000..4b14f9e2
--- /dev/null
+++ b/template_windows.go
@@ -0,0 +1,17 @@
+//+build windows
+
+package gomplate
+
+import (
+	"os"
+
+	"golang.org/x/sys/windows"
+)
+
+func isDirError(name string) *os.PathError {
+	return &os.PathError{
+		Op:   "open",
+		Path: name,
+		Err:  windows.ERROR_INVALID_HANDLE,
+	}
+}