Avoid discarded primary-type speculation and bump to 3.1.0

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 0d5e5f8a-36d8-47aa-8526-9275e8355e2c

Author: Jordan Bolton

README.md

@@ -45,6 +45,13 @@ Cancellation still returns a `CommonError` wrapping `CancellationError`; polling
 
 The parser started off as a naive recursive descent parser with limited backtracking. It mostly followed the [official specification](https://docs.microsoft.com/en-us/powerquery-m/power-query-m-language-specification) released in October 2016. Deviations from the specification should be marked down in [specification.md](specification.md). A combinatorial parser has since been added which uses the naive parser as its base.
 
+`readType` skips impossible primary-type attempts before falling back to a
+primary expression. Strict primary-type reads retain their original diagnostics.
+Prediction also applies with custom parser objects, cancellation tokens, and tracing.
+Skipped attempts emit no traces or checkpoint calls; cancellation checks remain active.
+EOF and reads without an active context retain the speculative path.
+Return-type disambiguation is unchanged.
+
 ### Style
 
 This project uses [prettier](https://github.com/prettier/prettier) as the primary source of style enforcement. Additional style requirements are located in [style.md](style.md).

package-lock.json

@@ -1,6 +1,6 @@
 {
     "name": "@microsoft/powerquery-parser",
-    "version": "3.0.0",
+    "version": "3.1.0",
     "description": "A parser for the Power Query/M formula language.",
     "author": "Microsoft",
     "license": "MIT",

package.json

@@ -1922,6 +1922,21 @@ export async function readType(
 
     state.cancellationToken?.throwIfCancelled();
 
+    // Only readType discards a failed primary-type diagnostic in favor of an expression.
+    // Skipped attempts emit no traces or checkpoint calls; cancellation is still checked by the readers.
+    if (
+        state.currentContextNode !== undefined &&
+        state.tokenIndex < state.lexerSnapshot.tokens.length &&
+        !canStartPrimaryType(state)
+    ) {
+        trace.exit({
+            [NaiveTraceConstant.TokenIndex]: state.tokenIndex,
+            [TraceConstant.IsError]: true,
+        });
+
+        return parser.readPrimaryExpression(state, parser, trace.id);
+    }
+
     const triedReadPrimaryType: TriedReadPrimaryType = await tryReadPrimaryType(state, parser, trace.id);
 
     if (ResultUtils.isOk(triedReadPrimaryType)) {
@@ -2348,6 +2363,21 @@ export async function readFunctionType(
     return functionType;
 }
 
+function canStartPrimaryType(state: ParseState): boolean {
+    const token: Token.Token | undefined = state.lexerSnapshot.tokens[state.tokenIndex];
+
+    return (
+        token !== undefined &&
+        (token.kind === TokenKind.LeftBracket ||
+            token.kind === TokenKind.LeftBrace ||
+            token.kind === TokenKind.KeywordType ||
+            token.kind === TokenKind.NullLiteral ||
+            (token.kind === TokenKind.Identifier &&
+                (token.data === Constant.LanguageConstant.Nullable ||
+                    ConstantUtils.isPrimitiveTypeConstant(token.data))))
+    );
+}
+
 async function tryReadPrimaryType(
     state: ParseState,
     parser: Parser,

src/powerquery-parser/parser/parsers/naiveParseSteps.ts

@@ -0,0 +1,233 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import "mocha";
+import { expect } from "chai";
+import { rejects } from "assert";
+
+import {
+    CommonError,
+    CounterCancellationToken,
+    DefaultSettings,
+    NoOpCancellationToken,
+    Parser,
+    Task,
+    TaskUtils,
+} from "../../..";
+import { Trace, TraceConstant, TraceManager } from "../../../powerquery-parser/common/trace";
+import { Ast } from "../../../powerquery-parser/language";
+
+// Strict primary-type reads still perform speculation. Only an unconsumed primitive-type
+// failure is discarded here, reproducing readType's fallback without using its prediction.
+async function readTypeWithoutPrediction(
+    state: Parser.ParseState,
+    parser: Parser.Parser,
+    correlationId: number | undefined,
+): Promise<Ast.TType> {
+    const tokenIndex: number = state.tokenIndex;
+
+    try {
+        return await parser.readPrimaryType(state, parser, correlationId);
+    } catch (error: unknown) {
+        if (
+            state.tokenIndex !== tokenIndex ||
+            !(
+                error instanceof Parser.ParseError.InvalidPrimitiveTypeError ||
+                error instanceof Parser.ParseError.ExpectedAnyTokenKindError
+            )
+        ) {
+            throw error;
+        }
+
+        return parser.readPrimaryExpression(state, parser, correlationId);
+    }
+}
+
+class RecordingTraceManager extends TraceManager {
+    public readonly events: { trace: Trace; message: string }[] = [];
+
+    public emit(trace: Trace, message: string): void {
+        this.events.push({ trace, message });
+    }
+}
+
+function newState(overrides?: Partial<Parser.ParseState>): Parser.ParseState {
+    const lex: Task.TriedLexTask = TaskUtils.tryLex(DefaultSettings, "unknown");
+    TaskUtils.assertIsLexStageOk(lex);
+    const state: Parser.ParseState = Parser.ParseStateUtils.newState(lex.lexerSnapshot, overrides);
+    Parser.ParseStateUtils.startContext(state, Ast.NodeKind.ListType);
+
+    return state;
+}
+
+describe("Parser primary-type prediction", () => {
+    for (const parser of [Parser.CombinatorialParserV2, Parser.RecursiveDescentParser]) {
+        for (const text of [
+            "let a = type unknown in a",
+            "let a = type (1 + 2) in a",
+            "let a = type @unknown in a",
+            "let a = type 1 in a",
+            "let a = type in a",
+            "let a = type table [x=",
+            "let a = type function (x as",
+            "let a = type function (x as number) as ",
+            "let a = type [x = number] in a",
+            "let a = type nullable number in a",
+            "(x as number) as nullable number => x",
+        ]) {
+            it(`preserves full parse results for ${text}`, async () => {
+                const actual: Task.TriedLexParseTask = await TaskUtils.tryLexParse(
+                    { ...DefaultSettings, parser },
+                    text,
+                );
+
+                const expected: Task.TriedLexParseTask = await TaskUtils.tryLexParse(
+                    { ...DefaultSettings, parser: { ...parser, readType: readTypeWithoutPrediction } },
+                    text,
+                );
+
+                expect(actual).to.deep.equal(expected);
+
+                expect("nodeIdMapCollection" in actual ? actual.nodeIdMapCollection : undefined).to.deep.equal(
+                    "nodeIdMapCollection" in expected ? expected.nodeIdMapCollection : undefined,
+                );
+
+                expect("parseState" in actual ? actual.parseState : undefined).to.deep.equal(
+                    "parseState" in expected ? expected.parseState : undefined,
+                );
+            });
+        }
+
+        it("skips speculation with a cancellation token and custom parser hooks", async () => {
+            const state: Parser.ParseState = newState({ cancellationToken: NoOpCancellationToken });
+            let primaryReads: number = 0;
+
+            const customParser: Parser.Parser = {
+                ...parser,
+                checkpoint: (): never => {
+                    throw new Error("The skipped attempt must not create a checkpoint");
+                },
+                restoreCheckpoint: (): never => {
+                    throw new Error("The skipped attempt must not restore a checkpoint");
+                },
+                readPrimaryExpression: async (
+                    state: Parser.ParseState,
+                    parser: Parser.Parser,
+                    correlationId: number | undefined,
+                ): Promise<Ast.TPrimaryExpression> => {
+                    primaryReads += 1;
+
+                    return await Parser.NaiveParseSteps.readPrimaryExpression(state, parser, correlationId);
+                },
+            };
+
+            const result: Ast.TType = await customParser.readType(state, customParser, undefined);
+            expect(result.kind).to.equal(Ast.NodeKind.IdentifierExpression);
+            expect(primaryReads).to.equal(1);
+        });
+
+        it("traces only executed reads with matching entries, exits, and correlations", async () => {
+            const traceManager: RecordingTraceManager = new RecordingTraceManager();
+            const state: Parser.ParseState = newState({ traceManager });
+            await parser.readType(state, parser, undefined);
+
+            const entries: Map<number, Trace> = new Map();
+            const exits: Set<number> = new Set();
+
+            for (const { trace, message } of traceManager.events) {
+                if (message === TraceConstant.Entry) {
+                    expect(entries.has(trace.id)).to.equal(false);
+
+                    if (trace.correlationId !== undefined) {
+                        expect(entries.has(trace.correlationId)).to.equal(true);
+                    }
+
+                    entries.set(trace.id, trace);
+                } else if (message === TraceConstant.Exit) {
+                    expect(entries.has(trace.id)).to.equal(true);
+                    expect(exits.has(trace.id)).to.equal(false);
+                    exits.add(trace.id);
+                }
+            }
+
+            const tasks: string[] = [...entries.values()].map((trace: Trace) => trace.task);
+            expect(tasks).to.include("readType");
+            expect(tasks).to.include("readPrimaryExpression");
+            expect(tasks).not.to.include("tryReadPrimaryType");
+            expect(tasks).not.to.include("tryReadPrimitiveType");
+            expect(exits.size).to.equal(entries.size);
+        });
+
+        it("honors cancellation before prediction", async () => {
+            const state: Parser.ParseState = newState({ cancellationToken: new CounterCancellationToken(1) });
+            const tokenIndex: number = state.tokenIndex;
+            const idCounter: number = state.contextState.idCounter;
+            await rejects(parser.readType(state, parser, undefined), CommonError.CancellationError);
+            expect(state.tokenIndex).to.equal(tokenIndex);
+            expect(state.contextState.idCounter).to.equal(idCounter);
+        });
+
+        it("honors cancellation when entering the primary-expression fallback", async () => {
+            const state: Parser.ParseState = newState({ cancellationToken: new CounterCancellationToken(2) });
+            let enteredFallback: boolean = false;
+
+            const customParser: Parser.Parser = {
+                ...parser,
+                readPrimaryExpression: async (
+                    state: Parser.ParseState,
+                    parser: Parser.Parser,
+                    correlationId: number | undefined,
+                ): Promise<Ast.TPrimaryExpression> => {
+                    enteredFallback = true;
+
+                    return await Parser.NaiveParseSteps.readPrimaryExpression(state, parser, correlationId);
+                },
+            };
+
+            await rejects(customParser.readType(state, customParser, undefined), CommonError.CancellationError);
+            expect(enteredFallback).to.equal(true);
+            expect(state.tokenIndex).to.equal(0);
+        });
+
+        it("retains strict primitive-type diagnostics and their checkpoint hooks", async () => {
+            let checkpoints: number = 0;
+
+            const customParser: Parser.Parser = {
+                ...parser,
+                checkpoint: (state: Parser.ParseState): Promise<Parser.ParseStateCheckpoint> => {
+                    checkpoints += 1;
+
+                    return parser.checkpoint(state);
+                },
+            };
+
+            await rejects(
+                customParser.readPrimaryType(newState(), customParser, undefined),
+                Parser.ParseError.InvalidPrimitiveTypeError,
+            );
+
+            expect(checkpoints).to.be.greaterThan(0);
+        });
+
+        it("retains EOF rollback and clears the stale cached token", async () => {
+            const state: Parser.ParseState = newState();
+            state.tokenIndex = state.lexerSnapshot.tokens.length;
+            state.currentTokenKind = undefined;
+            expect(state.currentToken).not.to.equal(undefined);
+            let checkpoints: number = 0;
+
+            const customParser: Parser.Parser = {
+                ...parser,
+                checkpoint: (state: Parser.ParseState): Promise<Parser.ParseStateCheckpoint> => {
+                    checkpoints += 1;
+
+                    return parser.checkpoint(state);
+                },
+            };
+
+            await rejects(customParser.readType(state, customParser, undefined));
+            expect(checkpoints).to.be.greaterThan(0);
+            expect(state.currentToken).to.equal(undefined);
+        });
+    }
+});