hcl/spectests: run the spec testsuite as part of "go test"

Although the spec testsuite and associated harness is designed to be
usable by other implementations of HCL not written in Go, it's convenient
to run it as part of our own "go test" test suite here so there isn't
an additional thing to run on each change.

To achieve this, the new package hcl/spectests will build both hcldec and
hclspecsuite from latest source and then run the latter to execute the
test suite, capturing the output and converting it (sloppily) into
testing.T method calls to produce something vaguely reasonable.

Other than the small amount of "parsing" to make it look in the output
like a normal Go test, there's nothing special going on here and so it's
still valid to run the spec suite manually with a build of hcldec from
this codebase, which should produce the same result.

3 files changed, 46 insertions, 18 deletions

diff --git a/cmd/hclspecsuite/log.go b/cmd/hclspecsuite/log.go
index f2af650..d5b5ae5 100644
--- a/cmd/hclspecsuite/log.go
+++ b/cmd/hclspecsuite/log.go
@@ -1,3 +1,8 @@
 package main
 
-type LogCallback func(testName string, testFile *TestFile)
+import (
+	"github.com/hashicorp/hcl2/hcl"
+)
+
+type LogBeginCallback func(testName string, testFile *TestFile)
+type LogProblemsCallback func(testName string, testFile *TestFile, diags hcl.Diagnostics)
diff --git a/cmd/hclspecsuite/main.go b/cmd/hclspecsuite/main.go
index 7e1aacd..4db5119 100644
--- a/cmd/hclspecsuite/main.go
+++ b/cmd/hclspecsuite/main.go
@@ -32,27 +32,40 @@ func realMain(args []string) int {
 
 	parser := hclparse.NewParser()
 
+	color := terminal.IsTerminal(int(os.Stderr.Fd()))
+	w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
+	if err != nil {
+		w = 80
+	}
+	diagWr := hcl.NewDiagnosticTextWriter(os.Stderr, parser.Files(), uint(w), color)
+	var diagCount int
+
 	runner := &Runner{
 		parser:     parser,
 		hcldecPath: hcldecPath,
 		baseDir:    testsDir,
-		log: func(name string, file *TestFile) {
+		logBegin: func(name string, file *TestFile) {
+			fmt.Printf("- %s\n", name)
+		},
+		logProblems: func(name string, file *TestFile, diags hcl.Diagnostics) {
+			if len(diags) != 0 {
+				os.Stderr.WriteString("\n")
+				diagWr.WriteDiagnostics(diags)
+				diagCount += len(diags)
+			}
 			fmt.Printf("- %s\n", name)
 		},
 	}
 	diags := runner.Run()
 
 	if len(diags) != 0 {
-		os.Stderr.WriteString("\n")
-		color := terminal.IsTerminal(int(os.Stderr.Fd()))
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err != nil {
-			w = 80
-		}
-		diagWr := hcl.NewDiagnosticTextWriter(os.Stderr, parser.Files(), uint(w), color)
+		os.Stderr.WriteString("\n\n\n== Test harness problems:\n\n")
 		diagWr.WriteDiagnostics(diags)
-		return 2
+		diagCount += len(diags)
 	}
 
+	if diagCount > 0 {
+		return 2
+	}
 	return 0
 }
diff --git a/cmd/hclspecsuite/runner.go b/cmd/hclspecsuite/runner.go
index 448d41c..4810454 100644
--- a/cmd/hclspecsuite/runner.go
+++ b/cmd/hclspecsuite/runner.go
@@ -21,10 +21,11 @@ import (
 )
 
 type Runner struct {
-	parser     *hclparse.Parser
-	hcldecPath string
-	baseDir    string
-	log        LogCallback
+	parser      *hclparse.Parser
+	hcldecPath  string
+	baseDir     string
+	logBegin    LogBeginCallback
+	logProblems LogProblemsCallback
 }
 
 func (r *Runner) Run() hcl.Diagnostics {
@@ -82,14 +83,18 @@ func (r *Runner) runTest(filename string) hcl.Diagnostics {
 	tf, diags := r.LoadTestFile(filename)
 	if diags.HasErrors() {
 		// We'll still log, so it's clearer which test the diagnostics belong to.
-		if r.log != nil {
-			r.log(prettyName, nil)
+		if r.logBegin != nil {
+			r.logBegin(prettyName, nil)
+		}
+		if r.logProblems != nil {
+			r.logProblems(prettyName, nil, diags)
+			return nil // don't duplicate the diagnostics we already reported
 		}
 		return diags
 	}
 
-	if r.log != nil {
-		r.log(prettyName, tf)
+	if r.logBegin != nil {
+		r.logBegin(prettyName, tf)
 	}
 
 	basePath := filename[:len(filename)-2]
@@ -127,6 +132,11 @@ func (r *Runner) runTest(filename string) hcl.Diagnostics {
 		diags = append(diags, moreDiags...)
 	}
 
+	if r.logProblems != nil {
+		r.logProblems(prettyName, nil, diags)
+		return nil // don't duplicate the diagnostics we already reported
+	}
+
 	return diags
 }