fix: preserve line comment at end of file without trailing newline

Author: efredriksson

spec/lang/lexer/comments_spec.lua

@@ -122,4 +122,31 @@ describe("lexer", function()
       }, tokens[11].comments)
    end)
 
+   it("preserves line comment at end of file without trailing newline", function()
+      local tokens = tl.lex("-- a comment")
+      assert.same(1, #tokens)
+      assert.same("$EOF$", tokens[1].tk)
+      assert.same({
+         {text = "-- a comment", x = 1, y = 1}
+      }, tokens[1].comments)
+   end)
+
+   it("preserves empty line comment at end of file without trailing newline", function()
+      local tokens = tl.lex("--")
+      assert.same(1, #tokens)
+      assert.same("$EOF$", tokens[1].tk)
+      assert.same({
+         {text = "--", x = 1, y = 1}
+      }, tokens[1].comments)
+   end)
+
+   it("preserves line comment starting with [ at end of file without trailing newline", function()
+      local tokens = tl.lex("--[")
+      assert.same(1, #tokens)
+      assert.same("$EOF$", tokens[1].tk)
+      assert.same({
+         {text = "--[", x = 1, y = 1}
+      }, tokens[1].comments)
+   end)
+
 end)

teal/lexer.lua

@@ -805,6 +805,10 @@ do
          end
       end
 
+      if state == "comment short" or state == "got --" or state == "got --[" then
+         add_comment(input:sub(ti, #input))
+      end
+
       table.insert(tokens, { x = x + 1, y = y, tk = "$EOF$", kind = "$EOF$", comments = comments })
 
       return tokens, errs

teal/lexer.tl

@@ -805,6 +805,10 @@ do
          end
       end
 
+      if state == "comment short" or state == "got --" or state == "got --[" then
+         add_comment(input:sub(ti, #input))
+      end
+
       table.insert(tokens, { x = x + 1, y = y, tk = "$EOF$", kind = "$EOF$", comments = comments })
 
       return tokens, errs

tl.lua

@@ -13302,6 +13302,10 @@ do
          end
       end
 
+      if state == "comment short" or state == "got --" or state == "got --[" then
+         add_comment(input:sub(ti, #input))
+      end
+
       table.insert(tokens, { x = x + 1, y = y, tk = "$EOF$", kind = "$EOF$", comments = comments })
 
       return tokens, errs