cmd/hclspecsuite: Check for expected diagnostics

When a test file declares one or more expected diagnostics, we check those
instead of checking the result value. The severities and source ranges
must match.

We don't test the error messages themselves because they are not part of
the specification and may vary between implementations or, in future, be
translated into other languages.

2 files changed, 117 insertions, 28 deletions

diff --git a/cmd/hclspecsuite/diagnostics.go b/cmd/hclspecsuite/diagnostics.go
index 802c0cb..a559f1b 100644
--- a/cmd/hclspecsuite/diagnostics.go
+++ b/cmd/hclspecsuite/diagnostics.go
@@ -87,3 +87,22 @@ func decodeJSONDiagnostics(src []byte) hcl.Diagnostics {
 
 	return diags
 }
+
+func severityString(severity hcl.DiagnosticSeverity) string {
+	switch severity {
+	case hcl.DiagError:
+		return "error"
+	case hcl.DiagWarning:
+		return "warning"
+	default:
+		return "unsupported-severity"
+	}
+}
+
+func rangeString(rng hcl.Range) string {
+	return fmt.Sprintf(
+		"from line %d column %d byte %d to line %d column %d byte %d",
+		rng.Start.Line, rng.Start.Column, rng.Start.Byte,
+		rng.End.Line, rng.End.Column, rng.End.Byte,
+	)
+}
diff --git a/cmd/hclspecsuite/runner.go b/cmd/hclspecsuite/runner.go
index 977a3d7..e3d1235 100644
--- a/cmd/hclspecsuite/runner.go
+++ b/cmd/hclspecsuite/runner.go
@@ -169,44 +169,114 @@ func (r *Runner) runTestInput(specFilename, inputFilename string, tf *TestFile)
 
 	}
 
-	val, moreDiags := r.hcldecTransform(specFilename, inputFilename)
-	diags = append(diags, moreDiags...)
-	if moreDiags.HasErrors() {
-		// If hcldec failed then there's no point in continuing.
-		return diags
-	}
-
-	if errs := val.Type().TestConformance(tf.ResultType); len(errs) > 0 {
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Incorrect result type",
-			Detail: fmt.Sprintf(
-				"Input file %s produced %s, but was expecting %s.",
-				inputFilename, typeexpr.TypeString(val.Type()), typeexpr.TypeString(tf.ResultType),
-			),
-		})
-	}
+	val, transformDiags := r.hcldecTransform(specFilename, inputFilename)
+	if len(tf.ExpectedDiags) == 0 {
+		diags = append(diags, transformDiags...)
+		if transformDiags.HasErrors() {
+			// If hcldec failed then there's no point in continuing.
+			return diags
+		}
 
-	if tf.Result != cty.NilVal {
-		cmpVal, err := convert.Convert(tf.Result, tf.ResultType)
-		if err != nil {
+		if errs := val.Type().TestConformance(tf.ResultType); len(errs) > 0 {
 			diags = append(diags, &hcl.Diagnostic{
 				Severity: hcl.DiagError,
-				Summary:  "Incorrect type for result value",
+				Summary:  "Incorrect result type",
 				Detail: fmt.Sprintf(
-					"Result does not conform to the given result type: %s.", err,
+					"Input file %s produced %s, but was expecting %s.",
+					inputFilename, typeexpr.TypeString(val.Type()), typeexpr.TypeString(tf.ResultType),
 				),
-				Subject: &tf.ResultRange,
 			})
-		} else {
-			if !val.RawEquals(cmpVal) {
+		}
+
+		if tf.Result != cty.NilVal {
+			cmpVal, err := convert.Convert(tf.Result, tf.ResultType)
+			if err != nil {
+				diags = append(diags, &hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Incorrect type for result value",
+					Detail: fmt.Sprintf(
+						"Result does not conform to the given result type: %s.", err,
+					),
+					Subject: &tf.ResultRange,
+				})
+			} else {
+				if !val.RawEquals(cmpVal) {
+					diags = append(diags, &hcl.Diagnostic{
+						Severity: hcl.DiagError,
+						Summary:  "Incorrect result value",
+						Detail: fmt.Sprintf(
+							"Input file %s produced %#v, but was expecting %#v.",
+							inputFilename, val, tf.Result,
+						),
+					})
+				}
+			}
+		}
+	} else {
+		// We're expecting diagnostics, and so we'll need to correlate the
+		// severities and source ranges of our actual diagnostics against
+		// what we were expecting.
+		type DiagnosticEntry struct {
+			Severity hcl.DiagnosticSeverity
+			Range    hcl.Range
+		}
+		got := make(map[DiagnosticEntry]*hcl.Diagnostic)
+		want := make(map[DiagnosticEntry]hcl.Range)
+		for _, diag := range transformDiags {
+			if diag.Subject == nil {
+				// Sourceless diagnostics can never be expected, so we'll just
+				// pass these through as-is and assume they are hcldec
+				// operational errors.
+				diags = append(diags, diag)
+				continue
+			}
+			if diag.Subject.Filename != inputFilename {
+				// If the problem is for something other than the input file
+				// then it can't be expected.
+				diags = append(diags, diag)
+				continue
+			}
+			entry := DiagnosticEntry{
+				Severity: diag.Severity,
+				Range:    *diag.Subject,
+			}
+			got[entry] = diag
+		}
+		for _, e := range tf.ExpectedDiags {
+			e.Range.Filename = inputFilename // assumed here, since we don't allow any other filename to be expected
+			entry := DiagnosticEntry{
+				Severity: e.Severity,
+				Range:    e.Range,
+			}
+			want[entry] = e.DeclRange
+		}
+
+		for gotEntry, diag := range got {
+			if _, wanted := want[gotEntry]; !wanted {
+				// Pass through the diagnostic itself so the user can see what happened
+				diags = append(diags, diag)
+				diags = append(diags, &hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Unexpected diagnostic",
+					Detail: fmt.Sprintf(
+						"No %s diagnostic was expected %s. The unexpected diagnostic was shown above.",
+						severityString(gotEntry.Severity), rangeString(gotEntry.Range),
+					),
+					Subject: &gotEntry.Range,
+				})
+			}
+		}
+
+		for wantEntry, declRange := range want {
+			if _, gotted := got[wantEntry]; !gotted {
 				diags = append(diags, &hcl.Diagnostic{
 					Severity: hcl.DiagError,
-					Summary:  "Incorrect result value",
+					Summary:  "Missing expected diagnostic",
 					Detail: fmt.Sprintf(
-						"Input file %s produced %#v, but was expecting %#v.",
-						inputFilename, val, tf.Result,
+						"No %s diagnostic was generated %s.",
+						severityString(wantEntry.Severity), rangeString(wantEntry.Range),
 					),
+					Subject: &declRange,
 				})
 			}
 		}