gh-153569: remove duplicated tokenizer state

Author: pablogsal

Makefile.pre.in

@@ -394,7 +394,6 @@ PEGEN_OBJS=		\
 		Parser/peg_api.o
 
 TOKENIZER_OBJS=		\
-		Parser/lexer/buffer.o \
 		Parser/lexer/lexer.o \
 		Parser/lexer/number.o \
 		Parser/lexer/state.o \
@@ -410,7 +409,6 @@ PEGEN_HEADERS= \
 		$(srcdir)/Parser/string_parser.h
 
 TOKENIZER_HEADERS= \
-		Parser/lexer/buffer.h \
 		Parser/lexer/lexer.h \
 		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \

PCbuild/_freeze_module.vcxproj

@@ -181,7 +181,6 @@
     <ClCompile Include="..\Parser\action_helpers.c" />
     <ClCompile Include="..\Parser\string_parser.c" />
     <ClCompile Include="..\Parser\token.c" />
-    <ClCompile Include="..\Parser\lexer\buffer.c" />
     <ClCompile Include="..\Parser\lexer\state.c" />
     <ClCompile Include="..\Parser\lexer\lexer.c" />
     <ClCompile Include="..\Parser\lexer\number.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -469,9 +469,6 @@
     <ClCompile Include="..\Parser\lexer\string.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Parser\lexer\buffer.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Parser\lexer\state.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/pythoncore.vcxproj

@@ -422,7 +422,6 @@
     <ClInclude Include="..\Parser\lexer\state.h" />
     <ClInclude Include="..\Parser\lexer\lexer.h" />
     <ClInclude Include="..\Parser\lexer\lexer_internal.h" />
-    <ClInclude Include="..\Parser\lexer\buffer.h" />
     <ClInclude Include="..\Parser\tokenizer\reader.h" />
     <ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
     <ClInclude Include="..\Parser\tokenizer\source.h" />
@@ -591,7 +590,6 @@
     <ClCompile Include="..\Parser\lexer\lexer.c" />
     <ClCompile Include="..\Parser\lexer\number.c" />
     <ClCompile Include="..\Parser\lexer\string.c" />
-    <ClCompile Include="..\Parser\lexer\buffer.c" />
     <ClCompile Include="..\Parser\tokenizer\source.c" />
     <ClCompile Include="..\Parser\tokenizer\decoder.c" />
     <ClCompile Include="..\Parser\tokenizer\reader.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -330,9 +330,6 @@
     <ClInclude Include="..\Parser\lexer\lexer_internal.h">
       <Filter>Parser</Filter>
     </ClInclude>
-    <ClInclude Include="..\Parser\lexer\buffer.h">
-      <Filter>Parser</Filter>
-    </ClInclude>
     <ClInclude Include="..\Parser\tokenizer\reader.h">
       <Filter>Parser</Filter>
     </ClInclude>
@@ -1355,9 +1352,6 @@
     <ClCompile Include="..\Parser\lexer\state.c">
       <Filter>Parser</Filter>
     </ClCompile>
-    <ClCompile Include="..\Parser\lexer\buffer.c">
-      <Filter>Parser</Filter>
-    </ClCompile>
     <ClCompile Include="..\Parser\tokenizer\source.c">
       <Filter>Parser</Filter>
     </ClCompile>

Parser/lexer/buffer.c

@@ -7,7 +7,7 @@
 #include "../tokenizer/helpers.h"
 #include "../tokenizer/reader.h"
 
-/* Alternate tab spacing */
+#define TABSIZE 8
 #define ALTTABSIZE 1
 
 
@@ -45,11 +45,10 @@ _PyLexer_nextc(struct tok_state *tok)
     int rc;
     for (;;) {
         if (tok->cur != tok->inp) {
-            if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) {
+            if (tok->cur - tok->line_start >= INT_MAX) {
                 tok->done = E_COLUMNOVERFLOW;
                 return EOF;
             }
-            tok->col_offset++;
             return Py_CHARMASK(*tok->cur++); /* Fast path */
         }
         if (tok->done != E_OK) {
@@ -89,7 +88,6 @@ _PyLexer_backup(struct tok_state *tok, int c)
         if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) {
             Py_FatalError("tok_backup: wrong character");
         }
-        tok->col_offset--;
     }
 }
 
@@ -103,7 +101,7 @@ verify_identifier(struct tok_state *tok)
         return 1;
     }
     PyObject *s;
-    if (tok->input_error)
+    if (tok_failed(tok))
         return 0;
     s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
     if (s == NULL) {
@@ -180,7 +178,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
     const char *p_end = NULL;
   nextline:
     tok->start = NULL;
-    tok->starting_col_offset = -1;
+    tok->start_loc = (_PyTok_Loc){tok->lineno, -1};
     blankline = 0;
 
 
@@ -196,7 +194,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
                 col++, altcol++;
             }
             else if (c == '\t') {
-                col = (col / tok->tabsize + 1) * tok->tabsize;
+                col = (col / TABSIZE + 1) * TABSIZE;
                 altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
             }
             else if (c == '\014')  {/* Control-L (formfeed) */
@@ -221,15 +219,16 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
         }
         tok_backup(tok, c);
         if (c == '#' || c == '\n' || c == '\r') {
+            int interactive = _PyTok_ReaderIsInteractive(tok);
             /* Lines with only whitespace and/or comments
                shouldn't affect the indentation and are
                not passed to the parser as NEWLINE tokens,
                except *totally* empty lines in interactive
                mode, which signal the end of a command group. */
-            if (col == 0 && c == '\n' && tok->prompt != NULL) {
+            if (col == 0 && c == '\n' && interactive) {
                 blankline = 0; /* Let it through */
             }
-            else if (tok->prompt != NULL && tok->lineno == 1) {
+            else if (interactive && tok->lineno == 1) {
                 /* In interactive mode, if the first line contains
                    only spaces and/or a comment, let it through. */
                 blankline = 0;
@@ -284,7 +283,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
     }
 
     tok->start = tok->cur;
-    tok->starting_col_offset = tok->col_offset;
+    tok->start_loc = (_PyTok_Loc){
+        tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) : -1};
 
     /* Return pending indents/dedents */
     if (tok->pendin != 0) {
@@ -319,7 +319,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
 
     /* Set start of current token */
     tok->start = tok->cur == NULL ? NULL : tok->cur - 1;
-    tok->starting_col_offset = tok->col_offset - 1;
+    tok->start_loc = (_PyTok_Loc){
+        tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) - 1 : -1};
 
     /* Skip comment, unless it's a type comment */
     if (c == '#') {
@@ -350,7 +351,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
 
         if (tok->type_comments) {
             p = tok->start;
-            current_starting_col_offset = tok->starting_col_offset;
+            current_starting_col_offset = tok->start_loc.byte_col;
             prefix = type_comment_prefix;
             while (*prefix && p < tok->cur) {
                 if (*prefix == ' ') {
@@ -394,11 +395,15 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
                         tok_nextc(tok);
                         tok->atbol = 1;
                     }
-                    return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset);
+                    return MAKE_TYPE_COMMENT_TOKEN(
+                        TYPE_IGNORE, ignore_end_col_offset,
+                        _PyLexer_ByteColumn(tok));
                 } else {
                     p_start = type_start;
                     p_end = tok->cur;
-                    return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset);
+                    return MAKE_TYPE_COMMENT_TOKEN(
+                        TYPE_COMMENT, current_starting_col_offset,
+                        _PyLexer_ByteColumn(tok));
                 }
             }
         }
@@ -719,7 +724,7 @@ int
 _PyTokenizer_Get(struct tok_state *tok, struct token *token)
 {
     int result = tok_get(tok, token);
-    if (tok->input_error) {
+    if (tok_failed(tok)) {
         result = ERRORTOKEN;
     }
     return result;

Parser/lexer/buffer.h

@@ -1,6 +1,7 @@
 #ifndef _PY_LEXER_INTERNAL_H_
 #define _PY_LEXER_INTERNAL_H_
 
+#include "errcode.h"
 #include "lexer.h"
 
 #define is_potential_identifier_start(c) (\
@@ -44,6 +45,13 @@ TOK_NEXT_MODE(struct tok_state *tok)
 #define tok_nextc _PyLexer_nextc
 #define tok_backup _PyLexer_backup
 
+static inline int
+tok_failed(const struct tok_state *tok)
+{
+    return tok->done != E_OK && tok->done != E_EOF &&
+           tok->done != E_INTERACT_STOP;
+}
+
 int _PyLexer_nextc(struct tok_state *);
 void _PyLexer_backup(struct tok_state *, int);
 void _PyLexer_update_ftstring_expr(struct tok_state *, char);

Parser/lexer/lexer.c

@@ -6,9 +6,6 @@
 #include "state.h"
 #include "../tokenizer/reader.h"
 
-/* Never change this */
-#define TABSIZE 8
-
 /* Create and initialize a new tok_state structure */
 struct tok_state *
 _PyTokenizer_tok_new(void)
@@ -22,31 +19,21 @@ _PyTokenizer_tok_new(void)
     }
 
     tok->buf = tok->cur = tok->inp = NULL;
-    tok->fp_interactive = 0;
-    tok->interactive_src_start = NULL;
-    tok->interactive_src_end = NULL;
     tok->start = NULL;
     tok->done = E_OK;
     tok->fp = NULL;
-    tok->tabsize = TABSIZE;
     tok->indent = 0;
     tok->indstack[0] = 0;
     tok->atbol = 1;
     tok->pendin = 0;
-    tok->prompt = NULL;
     tok->lineno = 0;
-    tok->starting_col_offset = -1;
-    tok->col_offset = -1;
+    tok->start_loc = (_PyTok_Loc){-1, -1};
     tok->level = 0;
     tok->altindstack[0] = 0;
-    tok->input_error = 0;
     tok->encoding = NULL;
     tok->filename = NULL;
     tok->module = NULL;
     tok->type_comments = 0;
-    tok->interactive_underflow = IUNDERFLOW_NORMAL;
-    tok->str = NULL;
-    tok->report_warnings = 1;
     tok->tok_extra_tokens = 0;
     tok->comment_newline = 0;
     tok->implicit_newline = 0;
@@ -100,13 +87,12 @@ _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const
 {
     token->level = tok->level;
     token->span = _PyLexer_BufferSpan(tok, start, end);
-    int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;
-    token->start_loc = (_PyTok_Loc){lineno, -1};
-    token->end_loc = (_PyTok_Loc){tok->lineno, -1};
-
     if (start != NULL && end != NULL) {
-        token->start_loc.byte_col = tok->starting_col_offset;
-        token->end_loc.byte_col = tok->col_offset;
+        token->start_loc = tok->start_loc;
+        token->end_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};
+    }
+    else {
+        token->start_loc = token->end_loc = (_PyTok_Loc){tok->lineno, -1};
     }
     return type;
 }