From d8175ddf949a5dcd49cf9879fe971c9da803407e Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Sat, 29 Aug 2020 11:16:40 -0400 Subject: Only open output files when necessary Signed-off-by: Dave Henderson --- go.mod | 1 + gomplate_test.go | 4 +- internal/iohelpers/readers.go | 48 +++++ internal/iohelpers/readers_test.go | 49 ++++++ internal/iohelpers/writers.go | 214 +++++++++++++++++++++++ internal/iohelpers/writers_test.go | 159 +++++++++++++++++ internal/tests/integration/inputdir_test.go | 4 - internal/tests/integration/inputdir_unix_test.go | 70 ++++++++ internal/writers/writers.go | 171 ------------------ internal/writers/writers_test.go | 113 ------------ template.go | 27 +-- template_test.go | 58 +++++- template_unix.go | 17 ++ template_windows.go | 17 ++ 14 files changed, 643 insertions(+), 309 deletions(-) create mode 100644 internal/iohelpers/readers.go create mode 100644 internal/iohelpers/readers_test.go create mode 100644 internal/iohelpers/writers.go create mode 100644 internal/iohelpers/writers_test.go create mode 100644 internal/tests/integration/inputdir_unix_test.go delete mode 100644 internal/writers/writers.go delete mode 100644 internal/writers/writers_test.go create mode 100644 template_unix.go create mode 100644 template_windows.go diff --git a/go.mod b/go.mod index 7db66783..6f9dfcc4 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/zealic/xignore v0.3.3 gocloud.dev v0.20.0 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de + golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f gopkg.in/src-d/go-billy.v4 v4.3.2 gopkg.in/src-d/go-git.v4 v4.13.1 diff --git a/gomplate_test.go b/gomplate_test.go index 0142aae0..1d569958 100644 --- a/gomplate_test.go +++ b/gomplate_test.go @@ -16,7 +16,7 @@ import ( "github.com/hairyhenderson/gomplate/v3/conv" "github.com/hairyhenderson/gomplate/v3/data" "github.com/hairyhenderson/gomplate/v3/env" - "github.com/hairyhenderson/gomplate/v3/internal/writers" + "github.com/hairyhenderson/gomplate/v3/internal/iohelpers" "github.com/stretchr/testify/assert" ) @@ -158,7 +158,7 @@ func TestCustomDelim(t *testing.T) { func TestRunTemplates(t *testing.T) { defer func() { Stdout = os.Stdout }() buf := &bytes.Buffer{} - Stdout = &writers.NopCloser{Writer: buf} + Stdout = &iohelpers.NopCloser{Writer: buf} config := &Config{Input: "foo", OutputFiles: []string{"-"}} err := RunTemplates(config) assert.NoError(t, err) diff --git a/internal/iohelpers/readers.go b/internal/iohelpers/readers.go new file mode 100644 index 00000000..277d7ab9 --- /dev/null +++ b/internal/iohelpers/readers.go @@ -0,0 +1,48 @@ +package iohelpers + +import ( + "io" + "sync" +) + +// LazyReadCloser provides an interface to a ReadCloser that will open on the +// first access. The wrapped io.ReadCloser must be provided by 'open'. +func LazyReadCloser(open func() (io.ReadCloser, error)) io.ReadCloser { + return &lazyReadCloser{ + opened: sync.Once{}, + open: open, + } +} + +type lazyReadCloser struct { + opened sync.Once + r io.ReadCloser + // caches the error that came from open(), if any + openErr error + open func() (io.ReadCloser, error) +} + +var _ io.ReadCloser = (*lazyReadCloser)(nil) + +func (l *lazyReadCloser) openReader() (r io.ReadCloser, err error) { + l.opened.Do(func() { + l.r, l.openErr = l.open() + }) + return l.r, l.openErr +} + +func (l *lazyReadCloser) Close() error { + r, err := l.openReader() + if err != nil { + return err + } + return r.Close() +} + +func (l *lazyReadCloser) Read(p []byte) (n int, err error) { + r, err := l.openReader() + if err != nil { + return 0, err + } + return r.Read(p) +} diff --git a/internal/iohelpers/readers_test.go b/internal/iohelpers/readers_test.go new file mode 100644 index 00000000..ef300ef8 --- /dev/null +++ b/internal/iohelpers/readers_test.go @@ -0,0 +1,49 @@ +package iohelpers + +import ( + "bytes" + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLazyReadCloser(t *testing.T) { + r := newBufferCloser(bytes.NewBufferString("hello world")) + opened := false + l, ok := LazyReadCloser(func() (io.ReadCloser, error) { + opened = true + return r, nil + }).(*lazyReadCloser) + assert.True(t, ok) + + assert.False(t, opened) + assert.Nil(t, l.r) + assert.False(t, r.closed) + + p := make([]byte, 5) + n, err := l.Read(p) + assert.NoError(t, err) + assert.True(t, opened) + assert.Equal(t, r, l.r) + assert.Equal(t, 5, n) + + err = l.Close() + assert.NoError(t, err) + assert.True(t, r.closed) + + // test error propagation + l = LazyReadCloser(func() (io.ReadCloser, error) { + return nil, os.ErrNotExist + }).(*lazyReadCloser) + + assert.Nil(t, l.r) + + p = make([]byte, 5) + _, err = l.Read(p) + assert.Error(t, err) + + err = l.Close() + assert.Error(t, err) +} diff --git a/internal/iohelpers/writers.go b/internal/iohelpers/writers.go new file mode 100644 index 00000000..0358bc43 --- /dev/null +++ b/internal/iohelpers/writers.go @@ -0,0 +1,214 @@ +package iohelpers + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "sync" +) + +type emptySkipper struct { + open func() (io.WriteCloser, error) + + // internal + w io.WriteCloser + buf *bytes.Buffer + nw bool +} + +// NewEmptySkipper creates an io.WriteCloser that will only start writing once a +// non-whitespace byte has been encountered. The wrapped io.WriteCloser must be +// provided by the `open` func. +func NewEmptySkipper(open func() (io.WriteCloser, error)) io.WriteCloser { + return &emptySkipper{ + w: nil, + buf: &bytes.Buffer{}, + nw: false, + open: open, + } +} + +func (f *emptySkipper) Write(p []byte) (n int, err error) { + if !f.nw { + if allWhitespace(p) { + // buffer the whitespace + return f.buf.Write(p) + } + + // first time around, so open the writer + f.nw = true + f.w, err = f.open() + if err != nil { + return 0, err + } + if f.w == nil { + return 0, errors.New("nil writer returned by open") + } + // empty the buffer into the wrapped writer + _, err = f.buf.WriteTo(f.w) + if err != nil { + return 0, err + } + } + + return f.w.Write(p) +} + +// Close - implements io.Closer +func (f *emptySkipper) Close() error { + if f.w != nil { + return f.w.Close() + } + return nil +} + +func allWhitespace(p []byte) bool { + for _, b := range p { + if b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\v' { + continue + } + return false + } + return true +} + +// NopCloser returns a WriteCloser with a no-op Close method wrapping +// the provided io.Writer. +type NopCloser struct { + io.Writer +} + +// Close - implements io.Closer +func (n *NopCloser) Close() error { + return nil +} + +var ( + _ io.WriteCloser = (*NopCloser)(nil) + _ io.WriteCloser = (*emptySkipper)(nil) + _ io.WriteCloser = (*sameSkipper)(nil) +) + +type sameSkipper struct { + open func() (io.WriteCloser, error) + + // internal + r *bufio.Reader + w io.WriteCloser + buf *bytes.Buffer + diff bool +} + +// SameSkipper creates an io.WriteCloser that will only start writing once a +// difference with the current output has been encountered. The wrapped +// io.WriteCloser must be provided by 'open'. +func SameSkipper(r io.Reader, open func() (io.WriteCloser, error)) io.WriteCloser { + br := bufio.NewReader(r) + return &sameSkipper{ + r: br, + w: nil, + buf: &bytes.Buffer{}, + diff: false, + open: open, + } +} + +// Write - writes to the buffer, until a difference with the output is found, +// then flushes and writes to the wrapped writer. +func (f *sameSkipper) Write(p []byte) (n int, err error) { + if !f.diff { + in := make([]byte, len(p)) + _, err := f.r.Read(in) + if err != nil && err != io.EOF { + return 0, fmt.Errorf("failed to read: %w", err) + } + if bytes.Equal(in, p) { + return f.buf.Write(p) + } + + f.diff = true + err = f.flush() + if err != nil { + return 0, err + } + } + return f.w.Write(p) +} + +func (f *sameSkipper) flush() (err error) { + if f.w == nil { + f.w, err = f.open() + if err != nil { + return err + } + if f.w == nil { + return fmt.Errorf("nil writer returned by open") + } + } + // empty the buffer into the wrapped writer + _, err = f.buf.WriteTo(f.w) + return err +} + +// Close - implements io.Closer +func (f *sameSkipper) Close() error { + // Check to see if we missed anything in the reader + if !f.diff { + n, err := f.r.Peek(1) + if len(n) > 0 || err != io.EOF { + err = f.flush() + if err != nil { + return fmt.Errorf("failed to flush on close: %w", err) + } + } + } + + if f.w != nil { + return f.w.Close() + } + return nil +} + +// LazyWriteCloser provides an interface to a WriteCloser that will open on the +// first access. The wrapped io.WriteCloser must be provided by 'open'. +func LazyWriteCloser(open func() (io.WriteCloser, error)) io.WriteCloser { + return &lazyWriteCloser{ + opened: sync.Once{}, + open: open, + } +} + +type lazyWriteCloser struct { + opened sync.Once + w io.WriteCloser + // caches the error that came from open(), if any + openErr error + open func() (io.WriteCloser, error) +} + +var _ io.WriteCloser = (*lazyWriteCloser)(nil) + +func (l *lazyWriteCloser) openWriter() (r io.WriteCloser, err error) { + l.opened.Do(func() { + l.w, l.openErr = l.open() + }) + return l.w, l.openErr +} + +func (l *lazyWriteCloser) Close() error { + w, err := l.openWriter() + if err != nil { + return err + } + return w.Close() +} + +func (l *lazyWriteCloser) Write(p []byte) (n int, err error) { + w, err := l.openWriter() + if err != nil { + return 0, err + } + return w.Write(p) +} diff --git a/internal/iohelpers/writers_test.go b/internal/iohelpers/writers_test.go new file mode 100644 index 00000000..d226a03e --- /dev/null +++ b/internal/iohelpers/writers_test.go @@ -0,0 +1,159 @@ +package iohelpers + +import ( + "bytes" + "fmt" + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAllWhitespace(t *testing.T) { + testdata := []struct { + in []byte + expected bool + }{ + {[]byte(" "), true}, + {[]byte("foo"), false}, + {[]byte(" \t\n\n\v\r\n"), true}, + {[]byte(" foo "), false}, + } + + for _, d := range testdata { + assert.Equal(t, d.expected, allWhitespace(d.in)) + } +} + +func TestEmptySkipper(t *testing.T) { + testdata := []struct { + in []byte + empty bool + }{ + {[]byte(" "), true}, + {[]byte("foo"), false}, + {[]byte(" \t\n\n\v\r\n"), true}, + {[]byte(" foo "), false}, + } + + for _, d := range testdata { + w := newBufferCloser(&bytes.Buffer{}) + opened := false + f, ok := NewEmptySkipper(func() (io.WriteCloser, error) { + opened = true + return w, nil + }).(*emptySkipper) + + assert.True(t, ok) + n, err := f.Write(d.in) + assert.NoError(t, err) + assert.Equal(t, len(d.in), n) + err = f.Close() + assert.NoError(t, err) + if d.empty { + assert.Nil(t, f.w) + assert.False(t, opened) + } else { + assert.NotNil(t, f.w) + assert.True(t, opened) + assert.EqualValues(t, d.in, w.Bytes()) + } + } +} + +func newBufferCloser(b *bytes.Buffer) *bufferCloser { + return &bufferCloser{b, false} +} + +type bufferCloser struct { + *bytes.Buffer + + closed bool +} + +func (b *bufferCloser) Close() error { + b.closed = true + return nil +} + +func TestSameSkipper(t *testing.T) { + testdata := []struct { + in []byte + out []byte + same bool + }{ + {[]byte(" "), []byte(" "), true}, + {[]byte("foo"), []byte("foo"), true}, + {[]byte("foo"), nil, false}, + {[]byte("foo"), []byte("bar"), false}, + {[]byte("foobar"), []byte("foo"), false}, + {[]byte("foo"), []byte("foobar"), false}, + } + + for _, d := range testdata { + t.Run(fmt.Sprintf("in:%q/out:%q/same:%v", d.in, d.out, d.same), func(t *testing.T) { + r := bytes.NewBuffer(d.out) + w := newBufferCloser(&bytes.Buffer{}) + opened := false + f, ok := SameSkipper(r, func() (io.WriteCloser, error) { + opened = true + return w, nil + }).(*sameSkipper) + assert.True(t, ok) + + n, err := f.Write(d.in) + assert.NoError(t, err) + assert.Equal(t, len(d.in), n) + err = f.Close() + assert.NoError(t, err) + if d.same { + assert.Nil(t, f.w) + assert.False(t, opened) + assert.Empty(t, w.Bytes()) + } else { + assert.NotNil(t, f.w) + assert.True(t, opened) + assert.EqualValues(t, d.in, w.Bytes()) + } + }) + } +} + +func TestLazyWriteCloser(t *testing.T) { + w := newBufferCloser(&bytes.Buffer{}) + opened := false + l, ok := LazyWriteCloser(func() (io.WriteCloser, error) { + opened = true + return w, nil + }).(*lazyWriteCloser) + assert.True(t, ok) + + assert.False(t, opened) + assert.Nil(t, l.w) + assert.False(t, w.closed) + + p := []byte("hello world") + n, err := l.Write(p) + assert.NoError(t, err) + assert.True(t, opened) + assert.Equal(t, 11, n) + + err = l.Close() + assert.NoError(t, err) + assert.True(t, w.closed) + + // test error propagation + l = LazyWriteCloser(func() (io.WriteCloser, error) { + return nil, os.ErrNotExist + }).(*lazyWriteCloser) + + assert.Nil(t, l.w) + + p = []byte("hello world") + _, err = l.Write(p) + assert.Error(t, err) + + err = l.Close() + assert.Error(t, err) +} diff --git a/internal/tests/integration/inputdir_test.go b/internal/tests/integration/inputdir_test.go index 3e3329da..ab791b48 100644 --- a/internal/tests/integration/inputdir_test.go +++ b/internal/tests/integration/inputdir_test.go @@ -43,10 +43,6 @@ out/{{ .in | strings.ReplaceAll $f (index .filemap $f) }}.out ) } -func (s *InputDirSuite) TearDownTest(c *C) { - s.tmpDir.Remove() -} - func (s *InputDirSuite) TestInputDir(c *C) { result := icmd.RunCommand(GomplateBin, "--input-dir", s.tmpDir.Join("in"), diff --git a/internal/tests/integration/inputdir_unix_test.go b/internal/tests/integration/inputdir_unix_test.go new file mode 100644 index 00000000..5a6cde18 --- /dev/null +++ b/internal/tests/integration/inputdir_unix_test.go @@ -0,0 +1,70 @@ +//+build integration +//+build !windows + +package integration + +import ( + "fmt" + "io/ioutil" + "math" + "os" + + . "gopkg.in/check.v1" + + "golang.org/x/sys/unix" + "gotest.tools/v3/assert" + "gotest.tools/v3/fs" + "gotest.tools/v3/icmd" +) + +func setFileUlimit(b uint64) error { + ulimit := unix.Rlimit{ + Cur: b, + Max: math.MaxInt64, + } + err := unix.Setrlimit(unix.RLIMIT_NOFILE, &ulimit) + return err +} + +func (s *InputDirSuite) TestInputDirRespectsUlimit(c *C) { + numfiles := 32 + flist := map[string]string{} + for i := 0; i < numfiles; i++ { + k := fmt.Sprintf("file_%d", i) + flist[k] = fmt.Sprintf("hello world %d\n", i) + } + testdir := fs.NewDir(c, "ulimittestfiles", + fs.WithDir("in", fs.WithFiles(flist)), + ) + defer testdir.Remove() + + // we need another ~11 fds for other various things, so we'd be guaranteed + // to hit the limit if we try to have all the input files open + // simultaneously + setFileUlimit(uint64(numfiles)) + defer setFileUlimit(8192) + + result := icmd.RunCmd(icmd.Command(GomplateBin, + "--input-dir", testdir.Join("in"), + "--output-dir", testdir.Join("out"), + ), func(c *icmd.Cmd) { + c.Dir = testdir.Path() + }) + setFileUlimit(8192) + result.Assert(c, icmd.Success) + + files, err := ioutil.ReadDir(testdir.Join("out")) + assert.NilError(c, err) + assert.Equal(c, numfiles, len(files)) + + for i := 0; i < numfiles; i++ { + f := testdir.Join("out", fmt.Sprintf("file_%d", i)) + _, err := os.Stat(f) + assert.NilError(c, err) + + content, err := ioutil.ReadFile(f) + assert.NilError(c, err) + expected := fmt.Sprintf("hello world %d\n", i) + assert.Equal(c, expected, string(content)) + } +} diff --git a/internal/writers/writers.go b/internal/writers/writers.go deleted file mode 100644 index b40f4e73..00000000 --- a/internal/writers/writers.go +++ /dev/null @@ -1,171 +0,0 @@ -package writers - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" -) - -type emptySkipper struct { - open func() (io.WriteCloser, error) - - // internal - w io.WriteCloser - buf *bytes.Buffer - nw bool -} - -// NewEmptySkipper creates an io.WriteCloser that will only start writing once a -// non-whitespace byte has been encountered. The wrapped io.WriteCloser must be -// provided by the `open` func. -func NewEmptySkipper(open func() (io.WriteCloser, error)) io.WriteCloser { - return &emptySkipper{ - w: nil, - buf: &bytes.Buffer{}, - nw: false, - open: open, - } -} - -func (f *emptySkipper) Write(p []byte) (n int, err error) { - if !f.nw { - if allWhitespace(p) { - // buffer the whitespace - return f.buf.Write(p) - } - - // first time around, so open the writer - f.nw = true - f.w, err = f.open() - if err != nil { - return 0, err - } - if f.w == nil { - return 0, errors.New("nil writer returned by open") - } - // empty the buffer into the wrapped writer - _, err = f.buf.WriteTo(f.w) - if err != nil { - return 0, err - } - } - - return f.w.Write(p) -} - -// Close - implements io.Closer -func (f *emptySkipper) Close() error { - if f.w != nil { - return f.w.Close() - } - return nil -} - -func allWhitespace(p []byte) bool { - for _, b := range p { - if b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\v' { - continue - } - return false - } - return true -} - -// NopCloser returns a WriteCloser with a no-op Close method wrapping -// the provided io.Writer. -type NopCloser struct { - io.Writer -} - -// Close - implements io.Closer -func (n *NopCloser) Close() error { - return nil -} - -var ( - _ io.WriteCloser = (*NopCloser)(nil) - _ io.WriteCloser = (*emptySkipper)(nil) - _ io.WriteCloser = (*sameSkipper)(nil) -) - -type sameSkipper struct { - open func() (io.WriteCloser, error) - - // internal - r *bufio.Reader - w io.WriteCloser - buf *bytes.Buffer - diff bool -} - -// SameSkipper creates an io.WriteCloser that will only start writing once a -// difference with the current output has been encountered. The wrapped -// io.WriteCloser must be provided by 'open'. -func SameSkipper(r io.Reader, open func() (io.WriteCloser, error)) io.WriteCloser { - br := bufio.NewReader(r) - return &sameSkipper{ - r: br, - w: nil, - buf: &bytes.Buffer{}, - diff: false, - open: open, - } -} - -// Write - writes to the buffer, until a difference with the output is found, -// then flushes and writes to the wrapped writer. -func (f *sameSkipper) Write(p []byte) (n int, err error) { - if !f.diff { - in := make([]byte, len(p)) - _, err := f.r.Read(in) - if err != nil && err != io.EOF { - return 0, fmt.Errorf("failed to read: %w", err) - } - if bytes.Equal(in, p) { - return f.buf.Write(p) - } - - f.diff = true - err = f.flush() - if err != nil { - return 0, err - } - } - return f.w.Write(p) -} - -func (f *sameSkipper) flush() (err error) { - if f.w == nil { - f.w, err = f.open() - if err != nil { - return err - } - if f.w == nil { - return fmt.Errorf("nil writer returned by open") - } - } - // empty the buffer into the wrapped writer - _, err = f.buf.WriteTo(f.w) - return err -} - -// Close - implements io.Closer -func (f *sameSkipper) Close() error { - // Check to see if we missed anything in the reader - if !f.diff { - n, err := f.r.Peek(1) - if len(n) > 0 || err != io.EOF { - err = f.flush() - if err != nil { - return fmt.Errorf("failed to flush on close: %w", err) - } - } - } - - if f.w != nil { - return f.w.Close() - } - return nil -} diff --git a/internal/writers/writers_test.go b/internal/writers/writers_test.go deleted file mode 100644 index b3452fcd..00000000 --- a/internal/writers/writers_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package writers - -import ( - "bytes" - "fmt" - "io" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestAllWhitespace(t *testing.T) { - testdata := []struct { - in []byte - expected bool - }{ - {[]byte(" "), true}, - {[]byte("foo"), false}, - {[]byte(" \t\n\n\v\r\n"), true}, - {[]byte(" foo "), false}, - } - - for _, d := range testdata { - assert.Equal(t, d.expected, allWhitespace(d.in)) - } -} - -func TestEmptySkipper(t *testing.T) { - testdata := []struct { - in []byte - empty bool - }{ - {[]byte(" "), true}, - {[]byte("foo"), false}, - {[]byte(" \t\n\n\v\r\n"), true}, - {[]byte(" foo "), false}, - } - - for _, d := range testdata { - w := &bufferCloser{&bytes.Buffer{}} - opened := false - f, ok := NewEmptySkipper(func() (io.WriteCloser, error) { - opened = true - return w, nil - }).(*emptySkipper) - - assert.True(t, ok) - n, err := f.Write(d.in) - assert.NoError(t, err) - assert.Equal(t, len(d.in), n) - err = f.Close() - assert.NoError(t, err) - if d.empty { - assert.Nil(t, f.w) - assert.False(t, opened) - } else { - assert.NotNil(t, f.w) - assert.True(t, opened) - assert.EqualValues(t, d.in, w.Bytes()) - } - } -} - -type bufferCloser struct { - *bytes.Buffer -} - -func (b *bufferCloser) Close() error { - return nil -} - -func TestSameSkipper(t *testing.T) { - testdata := []struct { - in []byte - out []byte - same bool - }{ - {[]byte(" "), []byte(" "), true}, - {[]byte("foo"), []byte("foo"), true}, - {[]byte("foo"), nil, false}, - {[]byte("foo"), []byte("bar"), false}, - {[]byte("foobar"), []byte("foo"), false}, - {[]byte("foo"), []byte("foobar"), false}, - } - - for _, d := range testdata { - t.Run(fmt.Sprintf("in:%q/out:%q/same:%v", d.in, d.out, d.same), func(t *testing.T) { - r := bytes.NewBuffer(d.out) - w := &bufferCloser{&bytes.Buffer{}} - opened := false - f, ok := SameSkipper(r, func() (io.WriteCloser, error) { - opened = true - return w, nil - }).(*sameSkipper) - assert.True(t, ok) - - n, err := f.Write(d.in) - assert.NoError(t, err) - assert.Equal(t, len(d.in), n) - err = f.Close() - assert.NoError(t, err) - if d.same { - assert.Nil(t, f.w) - assert.False(t, opened) - assert.Empty(t, w.Bytes()) - } else { - assert.NotNil(t, f.w) - assert.True(t, opened) - assert.EqualValues(t, d.in, w.Bytes()) - } - }) - } -} diff --git a/template.go b/template.go index b500ce73..448eff84 100644 --- a/template.go +++ b/template.go @@ -9,7 +9,7 @@ import ( "text/template" "github.com/hairyhenderson/gomplate/v3/internal/config" - "github.com/hairyhenderson/gomplate/v3/internal/writers" + "github.com/hairyhenderson/gomplate/v3/internal/iohelpers" "github.com/hairyhenderson/gomplate/v3/tmpl" "github.com/spf13/afero" @@ -101,7 +101,7 @@ func gatherTemplates(cfg *config.Config, outFileNamer func(string) (string, erro // --exec-pipe redirects standard out to the out pipe if cfg.OutWriter != nil { - Stdout = &writers.NopCloser{Writer: cfg.OutWriter} + Stdout = &iohelpers.NopCloser{Writer: cfg.OutWriter} } switch { @@ -227,7 +227,7 @@ func fileToTemplates(inFile, outFile string, mode os.FileMode, modeOverride bool func openOutFile(cfg *config.Config, filename string, mode os.FileMode, modeOverride bool) (out io.WriteCloser, err error) { if cfg.SuppressEmpty { - out = writers.NewEmptySkipper(func() (io.WriteCloser, error) { + out = iohelpers.NewEmptySkipper(func() (io.WriteCloser, error) { if filename == "-" { return Stdout, nil } @@ -260,14 +260,19 @@ func createOutFile(filename string, mode os.FileMode, modeOverride bool) (out io } // if the output file already exists, we'll use a SameSkipper - f, err := fs.OpenFile(filename, os.O_RDONLY, mode.Perm()) + fi, err := fs.Stat(filename) if err != nil { - // likely means the file just doesn't exist - open's error will be more useful - return open() + // likely means the file just doesn't exist - further errors will be more useful + return iohelpers.LazyWriteCloser(open), nil } - out = writers.SameSkipper(f, func() (io.WriteCloser, error) { - return open() - }) + if fi.IsDir() { + // error because this is a directory + return nil, isDirError(fi.Name()) + } + + out = iohelpers.SameSkipper(iohelpers.LazyReadCloser(func() (io.ReadCloser, error) { + return fs.OpenFile(filename, os.O_RDONLY, mode.Perm()) + }), open) return out, err } @@ -280,14 +285,14 @@ func readInput(filename string) (string, error) { } else { inFile, err = fs.OpenFile(filename, os.O_RDONLY, 0) if err != nil { - return "", fmt.Errorf("failed to open %s\n%v", filename, err) + return "", fmt.Errorf("failed to open %s: %w", filename, err) } // nolint: errcheck defer inFile.Close() } bytes, err := ioutil.ReadAll(inFile) if err != nil { - err = fmt.Errorf("read failed for %s\n%v", filename, err) + err = fmt.Errorf("read failed for %s: %w", filename, err) return "", err } return string(bytes), nil diff --git a/template_test.go b/template_test.go index b0ba3698..0f77f4c0 100644 --- a/template_test.go +++ b/template_test.go @@ -8,10 +8,11 @@ import ( "testing" "github.com/hairyhenderson/gomplate/v3/internal/config" - "github.com/hairyhenderson/gomplate/v3/internal/writers" + "github.com/hairyhenderson/gomplate/v3/internal/iohelpers" "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestReadInput(t *testing.T) { @@ -47,16 +48,20 @@ func TestOpenOutFile(t *testing.T) { _ = fs.Mkdir("/tmp", 0777) cfg := &config.Config{} - _, err := openOutFile(cfg, "/tmp/foo", 0644, false) + f, err := openOutFile(cfg, "/tmp/foo", 0644, false) assert.NoError(t, err) + + err = f.Close() + assert.NoError(t, err) + i, err := fs.Stat("/tmp/foo") assert.NoError(t, err) assert.Equal(t, os.FileMode(0644), i.Mode()) defer func() { Stdout = os.Stdout }() - Stdout = &writers.NopCloser{Writer: &bytes.Buffer{}} + Stdout = &iohelpers.NopCloser{Writer: &bytes.Buffer{}} - f, err := openOutFile(cfg, "-", 0644, false) + f, err = openOutFile(cfg, "-", 0644, false) assert.NoError(t, err) assert.Equal(t, Stdout, f) } @@ -120,8 +125,17 @@ func TestGatherTemplates(t *testing.T) { assert.Len(t, templates, 1) assert.Equal(t, "out", templates[0].targetPath) assert.Equal(t, os.FileMode(0644), templates[0].mode) - info, err := fs.Stat("out") + + // out file is created only on demand + _, err = fs.Stat("out") + assert.Error(t, err) + assert.True(t, os.IsNotExist(err)) + + _, err = templates[0].target.Write([]byte("hello world")) assert.NoError(t, err) + + info, err := fs.Stat("out") + require.NoError(t, err) assert.Equal(t, os.FileMode(0644), info.Mode()) fs.Remove("out") @@ -134,6 +148,10 @@ func TestGatherTemplates(t *testing.T) { assert.Equal(t, "bar", templates[0].contents) assert.NotEqual(t, Stdout, templates[0].target) assert.Equal(t, os.FileMode(0600), templates[0].mode) + + _, err = templates[0].target.Write([]byte("hello world")) + assert.NoError(t, err) + info, err = fs.Stat("out") assert.NoError(t, err) assert.Equal(t, os.FileMode(0600), info.Mode()) @@ -149,6 +167,10 @@ func TestGatherTemplates(t *testing.T) { assert.Equal(t, "bar", templates[0].contents) assert.NotEqual(t, Stdout, templates[0].target) assert.Equal(t, os.FileMode(0755), templates[0].mode) + + _, err = templates[0].target.Write([]byte("hello world")) + assert.NoError(t, err) + info, err = fs.Stat("out") assert.NoError(t, err) assert.Equal(t, os.FileMode(0755), info.Mode()) @@ -230,17 +252,26 @@ func TestProcessTemplates(t *testing.T) { }, } for _, in := range testdata { + actual, err := processTemplates(cfg, in.templates) assert.NoError(t, err) assert.Len(t, actual, len(in.templates)) for i, a := range actual { + current := in.templates[i] assert.Equal(t, in.contents[i], a.contents) - assert.Equal(t, in.templates[i].mode, a.mode) + assert.Equal(t, current.mode, a.mode) if len(in.targets) > 0 { assert.Equal(t, in.targets[i], a.target) } - if in.templates[i].targetPath != "-" { - info, err := fs.Stat(in.templates[i].targetPath) + if current.targetPath != "-" { + err = current.loadContents() + assert.NoError(t, err) + + n, err := current.target.Write([]byte("hello world")) + assert.NoError(t, err) + assert.Equal(t, 11, n) + + info, err := fs.Stat(current.targetPath) assert.NoError(t, err) assert.Equal(t, in.modes[i], info.Mode()) } @@ -248,3 +279,14 @@ func TestProcessTemplates(t *testing.T) { fs.Remove("out") } } + +func TestCreateOutFile(t *testing.T) { + origfs := fs + defer func() { fs = origfs }() + fs = afero.NewMemMapFs() + _ = fs.Mkdir("in", 0755) + + _, err := createOutFile("in", 0644, false) + assert.Error(t, err) + assert.IsType(t, &os.PathError{}, err) +} diff --git a/template_unix.go b/template_unix.go new file mode 100644 index 00000000..950b7fa7 --- /dev/null +++ b/template_unix.go @@ -0,0 +1,17 @@ +//+build !windows + +package gomplate + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func isDirError(name string) *os.PathError { + return &os.PathError{ + Op: "open", + Path: name, + Err: unix.EISDIR, + } +} diff --git a/template_windows.go b/template_windows.go new file mode 100644 index 00000000..4b14f9e2 --- /dev/null +++ b/template_windows.go @@ -0,0 +1,17 @@ +//+build windows + +package gomplate + +import ( + "os" + + "golang.org/x/sys/windows" +) + +func isDirError(name string) *os.PathError { + return &os.PathError{ + Op: "open", + Path: name, + Err: windows.ERROR_INVALID_HANDLE, + } +} -- cgit v1.2.3