Skip to content

Commit 0cd86c7

Browse files
msimacekmsimacek
committed
[GR-75717] Implement PyREPL support
PullRequest: graalpython/4873
2 parents 5bdcb92 + 769634c commit 0cd86c7

23 files changed

Lines changed: 479 additions & 91 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@ This changelog summarizes major changes between GraalVM versions of the Python
44
language runtime. The main focus is on user-observable behavior of the engine.
55

66
## Version 25.6
7-
- New Java-based `decimal` module with significanlty better performance
7+
* Add support for PyREPL and make it the default. It can be turned off by setting `PYTHON_BASIC_REPL=1` environment variable
8+
* New Java-based `decimal` module with significantly better performance
89

910
## Version 25.5
10-
- Added `select.poll()` on Linux and Mac OS
11-
- Better Windows console support and Windows version reporting
11+
* Added `select.poll()` on Linux and Mac OS
12+
* Better Windows console support and Windows version reporting
1213

1314
## Version 25.4
1415
* Remove BouncyCastle entirely. BouncyCastle was only needed for legacy RSA, DSA, and EC private keys versions 0 and 1. To support these from Python embeddings, BouncyCastle must now be explicitly enabled by adding the `org.graalvm.python:python-bouncycastle-support` Maven _and_ adding `bcprov-jdk18on`, `bcpkix-jdk18on`, and `bcutil-jdk18on` dependencies.

‎ci/python-bench.libsonnet‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,8 @@
260260
ruby: "==3.1.2",
261261
papi: "==5.5.1",
262262
},
263-
environment +: environment(self.os, self.arch) + {
263+
// Benchmarks may download build dependencies, such as Rust for pydantic-core.
264+
environment +: environment(self.os, self.arch) + $.overlay_imports.INTERNET_ACCESS_ENV + {
264265
BISECT_BENCHMARK_CONFIG: "bisect-benchmark.ini",
265266
ENABLE_POLYBENCH_HPC: "yes",
266267
POLYBENCH_HPC_EXTRA_HEADERS: "/cm/shared/apps/papi/papi-5.5.1/include",

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/AbstractParser.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ public enum Flags {
113113
INTERACTIVE_TERMINAL,
114114
ASYNC_HACKS,
115115
ALLOW_INCOMPLETE_INPUT,
116+
DONT_IMPLY_DEDENT,
116117
}
117118

118119
private static final String BARRY_AS_BDFL = "with Barry as BDFL, use '<>' instead of '!='";
@@ -358,7 +359,7 @@ public Token getAndInitializeToken() {
358359
if (startRule == InputType.SINGLE && token.type == Token.Kind.ENDMARKER && parsingStarted) {
359360
token.type = Token.Kind.NEWLINE;
360361
parsingStarted = false;
361-
if (tokenizer.getCurrentIndentIndex() > 0) {
362+
if (tokenizer.getCurrentIndentIndex() > 0 && !flags.contains(Flags.DONT_IMPLY_DEDENT)) {
362363
tokenizer.setPendingIndents(-tokenizer.getCurrentIndentIndex());
363364
tokenizer.setCurrentIndentIndex(0);
364365
}

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java‎

Lines changed: 70 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ private static void appendDecodedChars(StringBuilder builder, CharBuffer out) {
162162
private int repeatedRuns = 1;
163163
private boolean snaptshotStartup = false;
164164
private boolean warnDefaultEncoding = false;
165+
private final List<String> xOptions = new ArrayList<>();
165166
private int intMaxStrDigits = -1;
166167
private int cpuCount = -1;
167168
private boolean cpuCountXOptionSet = false;
@@ -462,6 +463,7 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
462463
case 'X':
463464
// CPython ignores unknown/unsupported -X options, so we can do that too
464465
String xOption = getShortOptionParameter(argumentIterator, remainder, 'X');
466+
xOptions.add(xOption);
465467
if ("warn_default_encoding".equals(xOption)) {
466468
warnDefaultEncoding = true;
467469
} else if (xOption.startsWith("int_max_str_digits")) {
@@ -857,6 +859,11 @@ protected void launch(Builder contextBuilder) {
857859

858860
origArgs.add(0, toAbsolutePath(executable));
859861
contextBuilder.option("python.OrigArgv", String.join(STRING_LIST_DELIMITER, origArgs));
862+
StringBuilder serializedXOptions = new StringBuilder();
863+
for (String xOption : xOptions) {
864+
serializedXOptions.append(xOption.length()).append(':').append(xOption);
865+
}
866+
contextBuilder.option("python.XOptions", serializedXOptions.toString());
860867

861868
// setting this to make sure our TopLevelExceptionHandler calls the excepthook
862869
// to print Python exceptions
@@ -929,8 +936,9 @@ protected void launch(Builder contextBuilder) {
929936
}
930937
}
931938

939+
boolean basicReplRequested = !tty || (!ignoreEnv && getBoolEnv("PYTHON_BASIC_REPL"));
932940
sysModule.putMember("_console_handler", consoleHandler);
933-
if (tty && !isolateFlag && (inspectFlag || (commandString == null && inputFile == null))) {
941+
if (basicReplRequested && tty && !isolateFlag && (inspectFlag || (commandString == null && inputFile == null))) {
934942
evalInternal(context, "import readline, rlcompleter");
935943
}
936944

@@ -949,8 +957,18 @@ protected void launch(Builder contextBuilder) {
949957
}
950958
}
951959
if ((tty && commandString == null && inputFile == null) || inspectFlag) {
960+
boolean inspecting = commandString != null || inputFile != null;
952961
inspectFlag = false;
953-
rc = readEvalPrint(context, consoleHandler, sysModule);
962+
runInteractiveHook(sysModule);
963+
if (inspecting) {
964+
// The inspected program may have changed os.environ since the launcher started.
965+
basicReplRequested = !tty || (!ignoreEnv && evalInternal(context, "import os; bool(os.environ.get('PYTHON_BASIC_REPL'))").asBoolean());
966+
}
967+
if (!basicReplRequested && canUsePyRepl(context)) {
968+
rc = readEvalPrintPyRepl(context, inspecting);
969+
} else {
970+
rc = readEvalPrint(context, consoleHandler, sysModule);
971+
}
954972
}
955973
} catch (RuntimeException e) {
956974
if (e.getMessage() != null && e.getMessage().contains("did not complete all polyglot threads")) {
@@ -1343,6 +1361,55 @@ private static ConsoleHandler createConsoleHandler(InputStream inStream, OutputS
13431361
}
13441362
}
13451363

1364+
private boolean canUsePyRepl(Context context) {
1365+
try {
1366+
return evalInternal(context, """
1367+
from _pyrepl.main import CAN_USE_PYREPL, FAIL_REASON
1368+
if not CAN_USE_PYREPL and FAIL_REASON:
1369+
import sys
1370+
from _pyrepl.trace import trace
1371+
trace(FAIL_REASON)
1372+
print(FAIL_REASON, file=sys.stderr)
1373+
CAN_USE_PYREPL""").asBoolean();
1374+
} catch (PolyglotException e) {
1375+
return false;
1376+
}
1377+
}
1378+
1379+
private int readEvalPrintPyRepl(Context context, boolean inspecting) {
1380+
String source;
1381+
if (inspecting) {
1382+
source = """
1383+
from _pyrepl.main import interactive_console
1384+
interactive_console()""";
1385+
} else {
1386+
source = """
1387+
import runpy
1388+
runpy._run_module_as_main('_pyrepl')""";
1389+
}
1390+
try {
1391+
evalInternal(context, source);
1392+
return 0;
1393+
} catch (PolyglotException e) {
1394+
if (e.isExit()) {
1395+
return e.getExitStatus();
1396+
}
1397+
throw e;
1398+
}
1399+
}
1400+
1401+
private static void runInteractiveHook(Value sysModule) {
1402+
Value hook = null;
1403+
try {
1404+
hook = sysModule.getMember("__interactivehook__");
1405+
} catch (PolyglotException e) {
1406+
// ignore
1407+
}
1408+
if (hook != null) {
1409+
hook.execute();
1410+
}
1411+
}
1412+
13461413
/**
13471414
* The read-eval-print loop, which can take input from a console, command line expression or a
13481415
* file. There are two ways the repl can terminate:
@@ -1355,15 +1422,6 @@ private static ConsoleHandler createConsoleHandler(InputStream inStream, OutputS
13551422
*/
13561423
private int readEvalPrint(Context context, ConsoleHandler consoleHandler, Value sysModule) {
13571424
try {
1358-
Value hook = null;
1359-
try {
1360-
hook = sysModule.getMember("__interactivehook__");
1361-
} catch (PolyglotException e) {
1362-
// ignore
1363-
}
1364-
if (hook != null) {
1365-
hook.execute();
1366-
}
13671425
while (true) { // processing inputs
13681426
boolean doEcho = doEcho(context);
13691427
String ps1 = doEcho ? sysModule.getMember("ps1").asString() : null;
@@ -1498,7 +1556,7 @@ private static boolean canSkipFromEval(String input) {
14981556
}
14991557

15001558
private Value evalInternal(Context context, String code) {
1501-
return context.eval(Source.newBuilder(getLanguageId(), code, "<internal>").internal(true).buildLiteral());
1559+
return context.eval(Source.newBuilder(getLanguageId(), code, "<internal>").internal(true).option("python.NewGlobals", "true").buildLiteral());
15021560
}
15031561

15041562
/**

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/JLineConsoleHandler.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,8 @@ public void initializeReadline(Value readlineModule) {
100100
try {
101101
// The grapheme capability probe consumes pending input, including commands sent through
102102
// a pseudo-terminal before readline is initialized (GR-80639).
103-
terminal = TerminalBuilder.builder().streams(inputStream, outputStream).system(true).graphemeCluster(false).build();
103+
// PyREPL also imports readline. Do not consume terminal input until readLine is used.
104+
terminal = TerminalBuilder.builder().streams(inputStream, outputStream).system(true).graphemeCluster(false).paused(true).build();
104105
} catch (IOException ex) {
105106
throw new RuntimeException("unexpected error opening console reader", ex);
106107
} finally {
@@ -236,7 +237,14 @@ public String readLine(String prompt) {
236237
if (lineBuffer.isEmpty()) {
237238
Object[] signalHandlers = stashSignalHandlers();
238239
try {
239-
String lines = reader.readLine(prompt);
240+
Terminal terminal = reader.getTerminal();
241+
String lines;
242+
terminal.resume();
243+
try {
244+
lines = reader.readLine(prompt);
245+
} finally {
246+
terminal.pause(true);
247+
}
240248
Collections.addAll(lineBuffer, lines.split("\n"));
241249
} catch (UserInterruptException e) {
242250
throw e;

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/compiler/UnparserTests.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,14 @@
4242

4343
import static org.junit.Assert.assertEquals;
4444

45+
import java.util.EnumSet;
46+
4547
import org.junit.Test;
4648

49+
import com.oracle.graal.python.PythonLanguage;
4750
import com.oracle.graal.python.compiler.Unparser;
4851
import com.oracle.graal.python.compiler.bytecode_dsl.BytecodeDSLCompiler;
52+
import com.oracle.graal.python.pegparser.AbstractParser;
4953
import com.oracle.graal.python.pegparser.InputType;
5054
import com.oracle.graal.python.pegparser.Parser;
5155
import com.oracle.graal.python.pegparser.ParserCallbacks;
@@ -93,7 +97,7 @@ public void testUnparseConstant() {
9397

9498
private static void checkRoundTrip(String source) {
9599
ParserCallbacks parserCallbacks = new TestParserCallbacksImpl();
96-
Parser parser = BytecodeDSLCompiler.createParser(source, parserCallbacks, InputType.EVAL, false, false);
100+
Parser parser = BytecodeDSLCompiler.createParser(source, parserCallbacks, InputType.EVAL, EnumSet.noneOf(AbstractParser.Flags.class), PythonLanguage.MINOR);
97101
ModTy.Expression result = (ModTy.Expression) parser.parse();
98102
assertEquals(source, unparse(result.body));
99103
}

‎graalpython/com.oracle.graal.python.test/src/tests/test_cmd_line.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,16 @@ def test_cpu_count_invalid(self):
9898
result = self.run_cpu_count(cpu_count_env=value)
9999
self.assertNotEqual(0, result.returncode)
100100

101+
def test_xoptions(self):
102+
result = subprocess.run(
103+
[sys.executable, '-Xa', '-Xb=c,d=e', '-Xc', '-Xa=overridden', '-X', '', '-c',
104+
"import sys; print(tuple(sys._xoptions[key] for key in ('a', 'b', 'c', '')))"],
105+
capture_output=True,
106+
text=True,
107+
)
108+
self.assertEqual(0, result.returncode, result)
109+
self.assertEqual("('overridden', 'c,d=e', True, True)", result.stdout.strip())
110+
101111
def test_stdin_script_exit_code(self):
102112
code = "import sys\nsys.exit(42)\n"
103113
result = subprocess.run([sys.executable], input=code, text=True)

‎graalpython/com.oracle.graal.python.test/src/tests/test_parser.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,20 @@ def test_compile_unrecognised_flags_message():
8282
assert False, "ValueError was not raised"
8383

8484

85+
def test_dont_imply_dedent():
86+
flags = 0x4000 | 0x200 # PyCF_ALLOW_INCOMPLETE_INPUT | PyCF_DONT_IMPLY_DEDENT
87+
source = "def foo():\n return 42"
88+
89+
try:
90+
compile(source, "<input>", "single", flags)
91+
except _IncompleteInputError:
92+
pass
93+
else:
94+
assert False, "_IncompleteInputError was not raised"
95+
96+
assert compile(source + "\n", "<input>", "single", flags)
97+
98+
8599
def test_lambdas_as_function_default_argument_values():
86100
globs = {}
87101
exec("""

0 commit comments

Comments
 (0)