From fd6e06ba477430ff092aa07cbc2415a26a150031 Mon Sep 17 00:00:00 2001
From: Katy Moe <katy@katy.moe>
Date: Thu, 2 Dec 2021 00:17:56 +0000
Subject: hclsyntax: test for invalid whitespace heredoc

A heredoc cannot be introduced by a line ending in whitespace.

For example, the first line of a heredoc whose delimeter is `FOO`
can be either '<<FOO' or '<<-FOO', but not '<<FOO ' or '<<-FOO `,
even if the last line ends in `FOO `.
---
 hclsyntax/scan_tokens_test.go | 166 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)

(limited to 'hclsyntax')

diff --git a/hclsyntax/scan_tokens_test.go b/hclsyntax/scan_tokens_test.go
index 720ac98..72dcc6f 100644
--- a/hclsyntax/scan_tokens_test.go
+++ b/hclsyntax/scan_tokens_test.go
@@ -1606,6 +1606,172 @@ EOF
 				},
 			},
 		},
+		{
+			`<<EOF 
+hello
+EOF
+`,
+			// `EOF ` is not a valid identifier
+			// so `<<EOF ` is not a valid TokenOHeredoc
+			[]Token{
+				{
+					Type:  TokenLessThan,
+					Bytes: []byte("<"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
+					},
+				},
+				{
+					Type:  TokenLessThan,
+					Bytes: []byte("<"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
+						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
+					},
+				},
+				{
+					Type:  TokenIdent,
+					Bytes: []byte("EOF"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 2, Line: 1, Column: 3},
+						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
+					},
+				},
+				{
+					Type:  TokenNewline,
+					Bytes: []byte("\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
+						End:   hcl.Pos{Byte: 7, Line: 2, Column: 1},
+					},
+				},
+				{
+					Type:  TokenIdent,
+					Bytes: []byte("hello"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 7, Line: 2, Column: 1},
+						End:   hcl.Pos{Byte: 12, Line: 2, Column: 6},
+					},
+				},
+				{
+					Type:  TokenNewline,
+					Bytes: []byte("\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 12, Line: 2, Column: 6},
+						End:   hcl.Pos{Byte: 13, Line: 3, Column: 1},
+					},
+				},
+				{
+					Type:  TokenIdent,
+					Bytes: []byte("EOF"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 13, Line: 3, Column: 1},
+						End:   hcl.Pos{Byte: 16, Line: 3, Column: 4},
+					},
+				},
+
+				{
+					Type:  TokenNewline,
+					Bytes: []byte("\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 16, Line: 3, Column: 4},
+						End:   hcl.Pos{Byte: 17, Line: 4, Column: 1},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 17, Line: 4, Column: 1},
+						End:   hcl.Pos{Byte: 17, Line: 4, Column: 1},
+					},
+				},
+			},
+		},
+		{
+			`<<EOF 
+hello
+EOF 
+`,
+			// `EOF ` is not a valid identifier
+			// so `<<EOF ` is not a valid TokenOHeredoc
+			[]Token{
+				{
+					Type:  TokenLessThan,
+					Bytes: []byte("<"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 1, Line: 1, Column: 2},
+					},
+				},
+				{
+					Type:  TokenLessThan,
+					Bytes: []byte("<"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 1, Line: 1, Column: 2},
+						End:   hcl.Pos{Byte: 2, Line: 1, Column: 3},
+					},
+				},
+				{
+					Type:  TokenIdent,
+					Bytes: []byte("EOF"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 2, Line: 1, Column: 3},
+						End:   hcl.Pos{Byte: 5, Line: 1, Column: 6},
+					},
+				},
+				{
+					Type:  TokenNewline,
+					Bytes: []byte("\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 6, Line: 1, Column: 7},
+						End:   hcl.Pos{Byte: 7, Line: 2, Column: 1},
+					},
+				},
+				{
+					Type:  TokenIdent,
+					Bytes: []byte("hello"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 7, Line: 2, Column: 1},
+						End:   hcl.Pos{Byte: 12, Line: 2, Column: 6},
+					},
+				},
+				{
+					Type:  TokenNewline,
+					Bytes: []byte("\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 12, Line: 2, Column: 6},
+						End:   hcl.Pos{Byte: 13, Line: 3, Column: 1},
+					},
+				},
+				{
+					Type:  TokenIdent,
+					Bytes: []byte("EOF"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 13, Line: 3, Column: 1},
+						End:   hcl.Pos{Byte: 16, Line: 3, Column: 4},
+					},
+				},
+
+				{
+					Type:  TokenNewline,
+					Bytes: []byte("\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 17, Line: 3, Column: 5},
+						End:   hcl.Pos{Byte: 18, Line: 4, Column: 1},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 18, Line: 4, Column: 1},
+						End:   hcl.Pos{Byte: 18, Line: 4, Column: 1},
+					},
+				},
+			},
+		},
 
 		// Combinations
 		{
-- 
cgit v1.2.3


From 1f6e39150176c0f19d079137edd1c26a2cc4e950 Mon Sep 17 00:00:00 2001
From: Katy Moe <katy@katy.moe>
Date: Wed, 2 Feb 2022 16:41:49 +0000
Subject: add further tests for indented heredocs

---
 hclsyntax/scan_tokens_test.go | 144 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 144 insertions(+)

(limited to 'hclsyntax')

diff --git a/hclsyntax/scan_tokens_test.go b/hclsyntax/scan_tokens_test.go
index 72dcc6f..c453738 100644
--- a/hclsyntax/scan_tokens_test.go
+++ b/hclsyntax/scan_tokens_test.go
@@ -1507,6 +1507,150 @@ EOT
 				},
 			},
 		},
+		{
+			`<<EOT
+  hello world
+EOT
+`,
+			[]Token{
+				{
+					Type:  TokenOHeredoc,
+					Bytes: []byte("<<EOT\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 6, Line: 2, Column: 1},
+					},
+				},
+				{
+					Type:  TokenStringLit,
+					Bytes: []byte("  hello world\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 6, Line: 2, Column: 1},
+						End:   hcl.Pos{Byte: 20, Line: 3, Column: 1},
+					},
+				},
+				{
+					Type:  TokenCHeredoc,
+					Bytes: []byte("EOT"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 20, Line: 3, Column: 1},
+						End:   hcl.Pos{Byte: 23, Line: 3, Column: 4},
+					},
+				},
+				{
+					Type:  TokenNewline,
+					Bytes: []byte("\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 23, Line: 3, Column: 4},
+						End:   hcl.Pos{Byte: 24, Line: 4, Column: 1},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 24, Line: 4, Column: 1},
+						End:   hcl.Pos{Byte: 24, Line: 4, Column: 1},
+					},
+				},
+			},
+		},
+		{
+			`<<-EOT
+  hello world
+EOT
+`,
+			[]Token{
+				{
+					Type:  TokenOHeredoc,
+					Bytes: []byte("<<-EOT\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 7, Line: 2, Column: 1},
+					},
+				},
+				{
+					Type:  TokenStringLit,
+					Bytes: []byte("  hello world\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 7, Line: 2, Column: 1},
+						End:   hcl.Pos{Byte: 21, Line: 3, Column: 1},
+					},
+				},
+				{
+					Type:  TokenCHeredoc,
+					Bytes: []byte("EOT"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 21, Line: 3, Column: 1},
+						End:   hcl.Pos{Byte: 24, Line: 3, Column: 4},
+					},
+				},
+				{
+					Type:  TokenNewline,
+					Bytes: []byte("\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 24, Line: 3, Column: 4},
+						End:   hcl.Pos{Byte: 25, Line: 4, Column: 1},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 25, Line: 4, Column: 1},
+						End:   hcl.Pos{Byte: 25, Line: 4, Column: 1},
+					},
+				},
+			},
+		},
+		{
+			`<<-EOT
+  hello world
+ EOT
+`,
+			[]Token{
+				{
+					Type:  TokenOHeredoc,
+					Bytes: []byte("<<-EOT\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   hcl.Pos{Byte: 7, Line: 2, Column: 1},
+					},
+				},
+				{
+					Type:  TokenStringLit,
+					Bytes: []byte("  hello world\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 7, Line: 2, Column: 1},
+						End:   hcl.Pos{Byte: 21, Line: 3, Column: 1},
+					},
+				},
+				{
+					Type:  TokenCHeredoc,
+					Bytes: []byte(" EOT"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 21, Line: 3, Column: 1},
+						End:   hcl.Pos{Byte: 25, Line: 3, Column: 5},
+					},
+				},
+				{
+					Type:  TokenNewline,
+					Bytes: []byte("\n"),
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 25, Line: 3, Column: 5},
+						End:   hcl.Pos{Byte: 26, Line: 4, Column: 1},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: hcl.Range{
+						Start: hcl.Pos{Byte: 26, Line: 4, Column: 1},
+						End:   hcl.Pos{Byte: 26, Line: 4, Column: 1},
+					},
+				},
+			},
+		},
 		{
 			`<<EOF
 ${<<-EOF
-- 
cgit v1.2.3