OpenSearch 3.x support - the scroll id belongs in the body, not the url

Author: pdphilip

.github/workflows/run-tests.yml

@@ -16,6 +16,7 @@ jobs:
       matrix:
         php: [ 8.3, 8.4 ]
         laravel: [ 13.*, 12.*, 11.* ]
+        opensearch: [ 2.19.6, 3.8.0 ]
         include:
           - laravel: 13.*
             testbench: 11.*
@@ -24,7 +25,7 @@ jobs:
           - laravel: 11.*
             testbench: 9.*
 
-    name: P${{ matrix.php }} - L${{ matrix.laravel }}
+    name: P${{ matrix.php }} - L${{ matrix.laravel }} - OS${{ matrix.opensearch }}
 
     env:
       BROADCAST_DRIVER: log
@@ -40,7 +41,10 @@ jobs:
 
     services:
       opensearch:
-        image: opensearchproject/opensearch:latest
+        # Always pinned, never ":latest". A silent major bump on the server side
+        # once turned every job red at the same time with no package change; a
+        # new server line should be a deliberate row here instead.
+        image: opensearchproject/opensearch:${{ matrix.opensearch }}
         env:
           discovery.type: single-node
           DISABLE_SECURITY_PLUGIN: "true"

phpstan-baseline.neon

@@ -1,23 +1,5 @@
 parameters:
 	ignoreErrors:
-		-
-			message: '''
-				#^Call to method __construct\(\) of deprecated class OpenSearch\\Helper\\Iterators\\SearchHitIterator\:
-				in 2\.4\.0 and will be removed in 3\.0\.0\.$#
-			'''
-			identifier: method.deprecatedClass
-			count: 1
-			path: src/Connection.php
-
-		-
-			message: '''
-				#^Call to method __construct\(\) of deprecated class OpenSearch\\Helper\\Iterators\\SearchResponseIterator\:
-				in 2\.4\.0 and will be removed in 3\.0\.0\.$#
-			'''
-			identifier: method.deprecatedClass
-			count: 2
-			path: src/Connection.php
-
 		-
 			message: '''
 				#^Call to method build\(\) of deprecated class OpenSearch\\ClientBuilder\:
@@ -126,24 +108,6 @@ parameters:
 			count: 1
 			path: src/Connection.php
 
-		-
-			message: '''
-				#^Instantiation of deprecated class OpenSearch\\Helper\\Iterators\\SearchHitIterator\:
-				in 2\.4\.0 and will be removed in 3\.0\.0\.$#
-			'''
-			identifier: new.deprecatedClass
-			count: 1
-			path: src/Connection.php
-
-		-
-			message: '''
-				#^Instantiation of deprecated class OpenSearch\\Helper\\Iterators\\SearchResponseIterator\:
-				in 2\.4\.0 and will be removed in 3\.0\.0\.$#
-			'''
-			identifier: new.deprecatedClass
-			count: 2
-			path: src/Connection.php
-
 		-
 			message: '''
 				#^Parameter \$builder of method PDPhilip\\OpenSearch\\Connection\:\:_buildAuth\(\) has typehint with deprecated class OpenSearch\\ClientBuilder\:

src/Connection.php

@@ -13,8 +13,6 @@
 use Illuminate\Support\Facades\Log;
 use OpenSearch\Client;
 use OpenSearch\ClientBuilder;
-use OpenSearch\Helper\Iterators\SearchHitIterator;
-use OpenSearch\Helper\Iterators\SearchResponseIterator;
 use OpenSearch\Namespaces\IndicesNamespace;
 use PDPhilip\Elasticsearch\Traits\HasOptions;
 use PDPhilip\OpenSearch\Exceptions\BulkInsertQueryException;
@@ -377,16 +375,30 @@ public function statement($query, $bindings = [], ?Blueprint $blueprint = null):
     public function searchResponseIterator($query, $scrollTimeout = '30s', $size = 100): Generator
     {
 
-        $scrollParams = [
+        $client = $this->openClient();
+
+        $response = $client->search([
             'scroll' => $scrollTimeout,
             'size' => $size, // Number of results per shard
             'index' => $query['index'],
             'body' => $query['body'],
-        ];
+        ]);
+
+        $scrollId = $response['_scroll_id'] ?? null;
+
+        try {
+            while (! empty($response['hits']['hits'])) {
+                yield $response;
 
-        $pages = new SearchResponseIterator($this->connection, $scrollParams);
-        foreach ($pages as $page) {
-            yield $page;
+                $response = $client->scroll([
+                    'scroll' => $scrollTimeout,
+                    'body' => ['scroll_id' => $scrollId],
+                ]);
+
+                $scrollId = $response['_scroll_id'] ?? $scrollId;
+            }
+        } finally {
+            $this->clearScroll($client, $scrollId);
         }
     }
 
@@ -406,27 +418,57 @@ public function cursor($query, $bindings = [], $useReadPdo = false, $scrollTimeo
         // We want to scroll by 1000 row chunks
         $query['body']['size'] = 1000;
 
-        $scrollParams = [
+        $client = $this->openClient();
+
+        $response = $client->search([
             'scroll' => $scrollTimeout,
             'index' => $query['index'],
             'body' => $query['body'],
-        ];
+        ]);
 
+        $scrollId = $response['_scroll_id'] ?? null;
         $count = 0;
-        $pages = new SearchResponseIterator($this->openClient(), $scrollParams);
-        $hits = new SearchHitIterator($pages);
 
-        foreach ($hits as $hit) {
-            $count++;
-            if ($count > $limit) {
-                break;
+        try {
+            while (! empty($response['hits']['hits'])) {
+                foreach ($response['hits']['hits'] as $hit) {
+                    $count++;
+                    if ($count > $limit) {
+                        return;
+                    }
+                    yield $hit;
+                }
+
+                $response = $client->scroll([
+                    'scroll' => $scrollTimeout,
+                    'body' => ['scroll_id' => $scrollId],
+                ]);
+
+                $scrollId = $response['_scroll_id'] ?? $scrollId;
             }
-            yield $hit;
+        } finally {
+            $this->clearScroll($client, $scrollId);
+        }
+    }
+
+    /**
+     * Release a scroll context.
+     *
+     * The id goes in the body, never the url. The client rawurlencodes a path
+     * scroll id and OpenSearch 3.x rejects the result ("Cannot parse scroll
+     * id"), which is what breaks every scroll on 3.x when the id happens to
+     * contain a character that needs encoding.
+     */
+    protected function clearScroll($client, ?string $scrollId): void
+    {
+        if ($scrollId === null) {
+            return;
         }
 
-        return (function () {
-            yield;
-        })();
+        $client->clearScroll([
+            'body' => ['scroll_id' => [$scrollId]],
+            'client' => ['ignore' => 404],
+        ]);
     }
 
     /**