Enhance executemany() to use multistatement (#798)

This pull request introduces a new `executemany_fallback` option to
improve how `executemany()` handles non-bulk DML statements, allowing
for client-side multi-statement batching. It also adds a new
`more_results()` method to the connection API, enhances documentation,
and expands test coverage for these features.

Author: methane

.github/workflows/tests.yaml

@@ -53,6 +53,11 @@ jobs:
         run: |
           pip install -r requirements.txt
 
+      - if: ${{ matrix.python-version == '3.12' || matrix.mariadb }}
+        name: Install SQLAlchemy
+        run: |
+          pip install "SQLAlchemy>=2,<3"
+
       - name: Run tests
         env:
           TESTDB: actions.cnf
@@ -99,10 +104,11 @@ jobs:
           wget https://github.com/django/django/archive/${DJANGO_VERSION}.tar.gz
           tar xf ${DJANGO_VERSION}.tar.gz
           cp ci/test_mysql.py django-${DJANGO_VERSION}/tests/
+          cp ci/test_mysql_executemany_multi.py django-${DJANGO_VERSION}/tests/
           cd django-${DJANGO_VERSION}
           pip install . -r tests/requirements/py3.txt
 
       - name: Run Django test
         run: |
           cd django-${DJANGO_VERSION}/tests/
-          PYTHONPATH=.. python3 ./runtests.py --settings=test_mysql
+          PYTHONPATH=.. python3 ./runtests.py --settings=test_mysql_executemany_multi

ci/test_mysql_executemany_multi.py

@@ -0,0 +1,5 @@
+from test_mysql import *  # noqa: F403
+
+
+for database in DATABASES.values():  # noqa: F405
+    database.setdefault("OPTIONS", {})["executemany_fallback"] = "multi"

doc/user_guide.rst

@@ -72,6 +72,7 @@ MySQL C API function mapping
  ``mysql_get_server_info()``        ``conn.get_server_info()``
  ``mysql_info()``                   ``conn.info()``
  ``mysql_insert_id()``              ``conn.insert_id()``
+ ``mysql_more_results()``           ``conn.more_results()``
  ``mysql_num_fields()``             ``result.num_fields()``
  ``mysql_num_rows()``               ``result.num_rows()``
  ``mysql_options()``                various options to ``_mysql.connect()``
@@ -321,6 +322,27 @@ connect(parameters...)
             overridden. Default: ``MySQLdb.cursors.Cursor``.  *This
             must be a keyword parameter.*
 
+         executemany_fallback
+            Controls how ``Cursor.executemany()`` handles statements that
+            cannot use the multi-row INSERT/REPLACE optimization. ``"loop"``
+            executes the statements one at a time and is the default.
+            ``"multi"`` may combine safe data manipulation statements into a
+            multi-statement query. If multi-statements are disabled or a query
+            is not eligible, execution silently falls back to ``"loop"``.
+
+            This is a connection option so it can be passed through, for
+            example, SQLAlchemy's ``connect_args`` or Django's database
+            ``OPTIONS``::
+
+                create_engine(
+                    "mysql+mysqldb://user:password@host/database",
+                    connect_args={"executemany_fallback": "multi"},
+                )
+
+                DATABASES["default"]["OPTIONS"]["executemany_fallback"] = "multi"
+
+            See ``executemany()`` below for batching and transaction details.
+
          use_unicode
             If True, CHAR and VARCHAR and TEXT columns are returned as
             Unicode strings, using the configured character set. It is
@@ -562,6 +584,31 @@ close()
       close the cursor when you are done with it and before creating a
       new one.
 
+executemany(operation, seq_of_params)
+      Executes an operation for every parameter set and returns the total
+      number of affected rows. Multi-row INSERT and REPLACE statements use
+      MySQLdb's existing single-statement ``VALUES`` rewrite whenever it
+      applies, independently of ``executemany_fallback``.
+
+      With ``executemany_fallback="multi"``, statements that do not match that
+      rewrite may instead be sent in multi-statement batches. This applies only
+      to SQL templates which begin with INSERT, REPLACE, UPDATE, or DELETE
+      (ignoring leading whitespace), contain no semicolon, and contain no
+      ``RETURNING`` clause. Other statements, including statements beginning
+      with a comment or ``WITH``, use the normal loop. The loop is also used
+      silently when the connection does not have multi-statements enabled.
+
+      Each batch is limited to 16000 encoded bytes, including separators, and
+      200 statements. A single rendered statement exceeding the byte limit is
+      executed alone. On successful completion, ``rowcount`` and the return
+      value are the sum of the affected-row counts for all statements.
+
+      Batching does not create an implicit transaction and is not atomic. If a
+      statement fails, statements before it may already have executed, while
+      statements after it do not execute. Applications needing all-or-nothing
+      behavior must manage a transaction explicitly; with autocommit enabled,
+      each statement may be committed independently.
+
 info()
       Returns some information about the last query. Normally
       you don't need to check this. If there are any MySQL

src/MySQLdb/_mysql.c

@@ -982,6 +982,28 @@ Returns 0 if there are more results; -1 if there are no more results\n\
 \n\
 Non-standard.\n\
 ";
+
+static const char _mysql_ConnectionObject_more_results__doc__[] =
+"Returns True if one or more results follow the current result of a\n\
+multi-statement query. This check does not advance to the next result.\n\
+\n\
+Non-standard.\n\
+";
+
+static PyObject *
+_mysql_ConnectionObject_more_results(
+    _mysql_ConnectionObject *self,
+    PyObject *noargs)
+{
+    int ret;
+    BEGIN_CONNECTION_OPERATION(self, return _mysql_Exception(self));
+    ret = mysql_more_results(&(self->connection));
+    END_CONNECTION_LOCK(self);
+    if (ret)
+        Py_RETURN_TRUE;
+    Py_RETURN_FALSE;
+}
+
 static PyObject *
 _mysql_ConnectionObject_next_result(
     _mysql_ConnectionObject *self,
@@ -2618,6 +2640,12 @@ static PyMethodDef _mysql_ConnectionObject_methods[] = {
         METH_NOARGS,
         _mysql_ConnectionObject_rollback__doc__
     },
+    {
+        "more_results",
+        (PyCFunction)_mysql_ConnectionObject_more_results,
+        METH_NOARGS,
+        _mysql_ConnectionObject_more_results__doc__
+    },
     {
         "next_result",
         (PyCFunction)_mysql_ConnectionObject_next_result,

src/MySQLdb/connections.py

@@ -53,6 +53,7 @@ class Connection(_mysql.connection):
     """MySQL Database Connection Object"""
 
     default_cursor = cursors.Cursor
+    executemany_fallback = "loop"
 
     def __init__(self, *args, **kwargs):
         """
@@ -123,6 +124,13 @@ class object, used to create cursors (keyword only)
             If True, enable multi statements for clients >= 4.1.
             Defaults to True.
 
+        :param str executemany_fallback:
+            Controls how ``Cursor.executemany()`` executes statements which
+            cannot use the multi-row INSERT/REPLACE optimization. ``"loop"``
+            executes each statement separately (the default), while
+            ``"multi"`` batches safe data manipulation statements into a
+            multi-statement query when multi statements are enabled.
+
         :param str ssl_mode:
             specify the security settings for connection to the server;
             see the MySQL documentation for more details
@@ -188,6 +196,13 @@ class object, used to create cursors (keyword only)
         use_unicode = kwargs2.pop("use_unicode", True)
         sql_mode = kwargs2.pop("sql_mode", "")
         self._binary_prefix = kwargs2.pop("binary_prefix", False)
+        executemany_fallback = kwargs2.pop(
+            "executemany_fallback", self.executemany_fallback
+        )
+        if executemany_fallback not in ("loop", "multi"):
+            raise ValueError(
+                "executemany_fallback must be either 'loop' or 'multi'"
+            )
 
         client_flag = kwargs.get("client_flag", 0)
         client_flag |= CLIENT.MULTI_RESULTS
@@ -203,6 +218,7 @@ class object, used to create cursors (keyword only)
         super().__init__(*args, **kwargs2)
 
         self.cursorclass = cursorclass
+        self.executemany_fallback = executemany_fallback
         self.encoders = {k: v for k, v in conv.items() if type(k) is not int}
         self._server_version = tuple(
             [numeric_part(n) for n in self.get_server_info().split(".")[:2]]