From 5e05dc9fb9ad3ada91466da64d20ffbf063ca93d Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Sat, 4 Feb 2023 15:03:03 -0500 Subject: replace afero module Signed-off-by: Dave Henderson --- template.go | 171 +++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 93 insertions(+), 78 deletions(-) (limited to 'template.go') diff --git a/template.go b/template.go index f9679b47..54095fdd 100644 --- a/template.go +++ b/template.go @@ -2,6 +2,7 @@ package gomplate import ( "context" + "errors" "fmt" "io" "io/fs" @@ -11,21 +12,20 @@ import ( "strings" "text/template" + "github.com/hack-pad/hackpadfs" "github.com/hairyhenderson/go-fsimpl" "github.com/hairyhenderson/gomplate/v4/internal/config" + "github.com/hairyhenderson/gomplate/v4/internal/datafs" "github.com/hairyhenderson/gomplate/v4/internal/iohelpers" "github.com/hairyhenderson/gomplate/v4/tmpl" - "github.com/spf13/afero" - "github.com/zealic/xignore" + // TODO: switch back if/when fs.FS support gets merged upstream + "github.com/hairyhenderson/xignore" ) // ignorefile name, like .gitignore const gomplateignore = ".gomplateignore" -// for overriding in tests -var aferoFS = afero.NewOsFs() - func addTmplFuncs(f template.FuncMap, root *template.Template, tctx interface{}, path string) { t := tmpl.New(root, tctx, path) tns := func() *tmpl.Template { return t } @@ -47,23 +47,6 @@ func copyFuncMap(funcMap template.FuncMap) template.FuncMap { return newFuncMap } -var fsProviderCtxKey = struct{}{} - -// ContextWithFSProvider returns a context with the given FSProvider. Should -// only be used in tests. -func ContextWithFSProvider(ctx context.Context, fsp fsimpl.FSProvider) context.Context { - return context.WithValue(ctx, fsProviderCtxKey, fsp) -} - -// FSProviderFromContext returns the FSProvider from the context, if any -func FSProviderFromContext(ctx context.Context) fsimpl.FSProvider { - if fsp, ok := ctx.Value(fsProviderCtxKey).(fsimpl.FSProvider); ok { - return fsp - } - - return nil -} - // parseTemplate - parses text as a Go template with the given name and options func parseTemplate(ctx context.Context, name, text string, funcs template.FuncMap, tmplctx interface{}, nested config.Templates, leftDelim, rightDelim string) (tmpl *template.Template, err error) { tmpl = template.New(name) @@ -89,7 +72,7 @@ func parseTemplate(ctx context.Context, name, text string, funcs template.FuncMa } func parseNestedTemplates(ctx context.Context, nested config.Templates, tmpl *template.Template) error { - fsp := FSProviderFromContext(ctx) + fsp := datafs.FSProviderFromContext(ctx) for alias, n := range nested { u := *n.URL @@ -106,6 +89,14 @@ func parseNestedTemplates(ctx context.Context, nested config.Templates, tmpl *te return fmt.Errorf("filesystem provider for %q unavailable: %w", &u, err) } + // TODO: maybe need to do something with root here? + if _, reldir := datafs.ResolveLocalPath(u.Path); reldir != "" && reldir != "." { + fsys, err = fs.Sub(fsys, reldir) + if err != nil { + return fmt.Errorf("sub filesystem for %q unavailable: %w", &u, err) + } + } + // inject context & header in case they're useful... fsys = fsimpl.WithContextFS(ctx, fsys) fsys = fsimpl.WithHeaderFS(n.Header, fsys) @@ -172,20 +163,22 @@ func parseNestedTemplate(_ context.Context, fsys fs.FS, alias, fname string, tmp // gatherTemplates - gather and prepare templates for rendering // nolint: gocyclo -func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer func(context.Context, string) (string, error)) (templates []Template, err error) { +func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer func(context.Context, string) (string, error)) ([]Template, error) { mode, modeOverride, err := cfg.GetMode() if err != nil { return nil, err } + var templates []Template + switch { // the arg-provided input string gets a special name case cfg.Input != "": // open the output file - no need to close it, as it will be closed by the // caller later - target, oerr := openOutFile(cfg.OutputFiles[0], 0755, mode, modeOverride, cfg.Stdout, cfg.SuppressEmpty) + target, oerr := openOutFile(ctx, cfg.OutputFiles[0], 0o755, mode, modeOverride, cfg.Stdout, cfg.SuppressEmpty) if oerr != nil { - return nil, oerr + return nil, fmt.Errorf("openOutFile: %w", oerr) } templates = []Template{{ @@ -197,14 +190,14 @@ func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer func( // input dirs presume output dirs are set too templates, err = walkDir(ctx, cfg, cfg.InputDir, outFileNamer, cfg.ExcludeGlob, mode, modeOverride) if err != nil { - return nil, err + return nil, fmt.Errorf("walkDir: %w", err) } case cfg.Input == "": templates = make([]Template, len(cfg.InputFiles)) - for i := range cfg.InputFiles { - templates[i], err = fileToTemplate(cfg, cfg.InputFiles[i], cfg.OutputFiles[i], mode, modeOverride) + for i, f := range cfg.InputFiles { + templates[i], err = fileToTemplate(ctx, cfg, f, cfg.OutputFiles[i], mode, modeOverride) if err != nil { - return nil, err + return nil, fmt.Errorf("fileToTemplate: %w", err) } } } @@ -216,48 +209,66 @@ func gatherTemplates(ctx context.Context, cfg *config.Config, outFileNamer func( // of .gomplateignore and exclude globs (if any), walk the input directory and create a list of // tplate objects, and an error, if any. func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer func(context.Context, string) (string, error), excludeGlob []string, mode os.FileMode, modeOverride bool) ([]Template, error) { - dir = filepath.Clean(dir) + dir = filepath.ToSlash(filepath.Clean(dir)) + + // we want a filesystem rooted at dir, for relative matching + fsys, err := datafs.FSysForPath(ctx, dir) + if err != nil { + return nil, fmt.Errorf("filesystem provider for %q unavailable: %w", dir, err) + } + + // we need dir to be relative to the root of fsys + // TODO: maybe need to do something with root here? + _, reldir := datafs.ResolveLocalPath(dir) - dirStat, err := aferoFS.Stat(dir) + subfsys, err := fs.Sub(fsys, reldir) if err != nil { - return nil, fmt.Errorf("couldn't stat %s: %w", dir, err) + return nil, fmt.Errorf("sub: %w", err) + } + + // just check . because fsys is subbed to dir already + dirStat, err := fs.Stat(subfsys, ".") + if err != nil { + return nil, fmt.Errorf("stat %q (%q): %w", dir, reldir, err) } dirMode := dirStat.Mode() templates := make([]Template, 0) - matcher := xignore.NewMatcher(aferoFS) + matcher := xignore.NewMatcher(subfsys) - // work around bug in xignore - a basedir of '.' doesn't work - basedir := dir - if basedir == "." { - basedir, _ = os.Getwd() - } - matches, err := matcher.Matches(basedir, &xignore.MatchesOptions{ + matches, err := matcher.Matches(".", &xignore.MatchesOptions{ Ignorefile: gomplateignore, Nested: true, // allow nested ignorefile AfterPatterns: excludeGlob, }) if err != nil { - return nil, fmt.Errorf("ignore matching failed for %s: %w", basedir, err) + return nil, fmt.Errorf("ignore matching failed for %s: %w", dir, err) } // Unmatched ignorefile rules's files - files := matches.UnmatchedFiles - for _, file := range files { - inFile := filepath.Join(dir, file) + for _, file := range matches.UnmatchedFiles { + // we want to pass an absolute (as much as possible) path to fileToTemplate + inPath := filepath.Join(dir, file) + inPath = filepath.ToSlash(inPath) + + // but outFileNamer expects only the filename itself outFile, err := outFileNamer(ctx, file) if err != nil { - return nil, err + return nil, fmt.Errorf("outFileNamer: %w", err) } - tpl, err := fileToTemplate(cfg, inFile, outFile, mode, modeOverride) + tpl, err := fileToTemplate(ctx, cfg, inPath, outFile, mode, modeOverride) if err != nil { - return nil, err + return nil, fmt.Errorf("fileToTemplate: %w", err) } - // Ensure file parent dirs - if err = aferoFS.MkdirAll(filepath.Dir(outFile), dirMode); err != nil { - return nil, err + // Ensure file parent dirs - use separate fsys for output file + outfsys, err := datafs.FSysForPath(ctx, outFile) + if err != nil { + return nil, fmt.Errorf("fsysForPath: %w", err) + } + if err = hackpadfs.MkdirAll(outfsys, filepath.Dir(outFile), dirMode); err != nil { + return nil, fmt.Errorf("mkdirAll %q: %w", outFile, err) } templates = append(templates, tpl) @@ -266,21 +277,26 @@ func walkDir(ctx context.Context, cfg *config.Config, dir string, outFileNamer f return templates, nil } -func fileToTemplate(cfg *config.Config, inFile, outFile string, mode os.FileMode, modeOverride bool) (Template, error) { +func fileToTemplate(ctx context.Context, cfg *config.Config, inFile, outFile string, mode os.FileMode, modeOverride bool) (Template, error) { source := "" //nolint:nestif if inFile == "-" { b, err := io.ReadAll(cfg.Stdin) if err != nil { - return Template{}, fmt.Errorf("failed to read from stdin: %w", err) + return Template{}, fmt.Errorf("read from stdin: %w", err) } source = string(b) } else { - si, err := aferoFS.Stat(inFile) + fsys, err := datafs.FSysForPath(ctx, inFile) + if err != nil { + return Template{}, fmt.Errorf("fsysForPath: %w", err) + } + + si, err := fs.Stat(fsys, inFile) if err != nil { - return Template{}, err + return Template{}, fmt.Errorf("stat %q: %w", inFile, err) } if mode == 0 { mode = si.Mode() @@ -288,17 +304,9 @@ func fileToTemplate(cfg *config.Config, inFile, outFile string, mode os.FileMode // we read the file and store in memory immediately, to prevent leaking // file descriptors. - f, err := aferoFS.OpenFile(inFile, os.O_RDONLY, 0) + b, err := fs.ReadFile(fsys, inFile) if err != nil { - return Template{}, fmt.Errorf("failed to open %s: %w", inFile, err) - } - - //nolint: errcheck - defer f.Close() - - b, err := io.ReadAll(f) - if err != nil { - return Template{}, fmt.Errorf("failed to read %s: %w", inFile, err) + return Template{}, fmt.Errorf("readAll %q: %w", inFile, err) } source = string(b) @@ -306,9 +314,9 @@ func fileToTemplate(cfg *config.Config, inFile, outFile string, mode os.FileMode // open the output file - no need to close it, as it will be closed by the // caller later - target, err := openOutFile(outFile, 0755, mode, modeOverride, cfg.Stdout, cfg.SuppressEmpty) + target, err := openOutFile(ctx, outFile, 0755, mode, modeOverride, cfg.Stdout, cfg.SuppressEmpty) if err != nil { - return Template{}, err + return Template{}, fmt.Errorf("openOutFile: %w", err) } tmpl := Template{ @@ -328,13 +336,13 @@ func fileToTemplate(cfg *config.Config, inFile, outFile string, mode os.FileMode // // TODO: the 'suppressEmpty' behaviour should be always enabled, in the next // major release (v4.x). -func openOutFile(filename string, dirMode, mode os.FileMode, modeOverride bool, stdout io.Writer, suppressEmpty bool) (out io.Writer, err error) { +func openOutFile(ctx context.Context, filename string, dirMode, mode os.FileMode, modeOverride bool, stdout io.Writer, suppressEmpty bool) (out io.Writer, err error) { if suppressEmpty { out = iohelpers.NewEmptySkipper(func() (io.Writer, error) { if filename == "-" { return stdout, nil } - return createOutFile(filename, dirMode, mode, modeOverride) + return createOutFile(ctx, filename, dirMode, mode, modeOverride) }) return out, nil } @@ -342,34 +350,41 @@ func openOutFile(filename string, dirMode, mode os.FileMode, modeOverride bool, if filename == "-" { return stdout, nil } - return createOutFile(filename, dirMode, mode, modeOverride) + return createOutFile(ctx, filename, dirMode, mode, modeOverride) } -func createOutFile(filename string, dirMode, mode os.FileMode, modeOverride bool) (out io.WriteCloser, err error) { +func createOutFile(ctx context.Context, filename string, dirMode, mode os.FileMode, modeOverride bool) (out io.WriteCloser, err error) { + // we only support writing out to local files for now + fsys, err := datafs.FSysForPath(ctx, filename) + if err != nil { + return nil, fmt.Errorf("fsysForPath: %w", err) + } + mode = iohelpers.NormalizeFileMode(mode.Perm()) if modeOverride { - err = aferoFS.Chmod(filename, mode) - if err != nil && !os.IsNotExist(err) { - return nil, fmt.Errorf("failed to chmod output file '%s' with mode %q: %w", filename, mode, err) + err = hackpadfs.Chmod(fsys, filename, mode) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("failed to chmod output file %q with mode %q: %w", filename, mode, err) } } open := func() (out io.WriteCloser, err error) { // Ensure file parent dirs - if err = aferoFS.MkdirAll(filepath.Dir(filename), dirMode); err != nil { - return nil, err + if err = hackpadfs.MkdirAll(fsys, filepath.Dir(filename), dirMode); err != nil { + return nil, fmt.Errorf("mkdirAll %q: %w", filename, err) } - out, err = aferoFS.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) + f, err := hackpadfs.OpenFile(fsys, filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) if err != nil { return out, fmt.Errorf("failed to open output file '%s' for writing: %w", filename, err) } + out = f.(io.WriteCloser) return out, err } // if the output file already exists, we'll use a SameSkipper - fi, err := aferoFS.Stat(filename) + fi, err := hackpadfs.Stat(fsys, filename) if err != nil { // likely means the file just doesn't exist - further errors will be more useful return iohelpers.LazyWriteCloser(open), nil @@ -380,7 +395,7 @@ func createOutFile(filename string, dirMode, mode os.FileMode, modeOverride bool } out = iohelpers.SameSkipper(iohelpers.LazyReadCloser(func() (io.ReadCloser, error) { - return aferoFS.OpenFile(filename, os.O_RDONLY, mode) + return hackpadfs.OpenFile(fsys, filename, os.O_RDONLY, mode) }), open) return out, err -- cgit v1.2.3