Skip to content

Commit f258a66

Browse files
authored
fix(clients): browseSynonyms pagination (CR-11727) (#6652)
1 parent b5d3fac commit f258a66

9 files changed

Lines changed: 595 additions & 8 deletions

File tree

‎clients/algoliasearch-client-javascript/packages/algoliasearch/__tests__/algoliasearch.common.test.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
DEFAULT_READ_TIMEOUT_BROWSER,
66
DEFAULT_WRITE_TIMEOUT_BROWSER,
77
} from '@algolia/client-common';
8+
import type { EndRequest, Requester } from '@algolia/client-common';
89
import type { EchoResponse } from '@algolia/requester-testing';
910

1011
import { browserEchoRequester } from '../../requester-testing/src/browserEchoRequester';
@@ -283,3 +284,44 @@ describe('init', () => {
283284
});
284285
});
285286
});
287+
288+
describe('browseSynonyms', () => {
289+
test('paginates until a non-full page and never re-fetches page 0 (CR-11727)', async () => {
290+
const hitsPerPage = 1000;
291+
const pagesRequested: number[] = [];
292+
293+
const paginatingRequester: Requester = {
294+
send(request: EndRequest) {
295+
const body = (typeof request.data === 'string' ? JSON.parse(request.data) : {}) as { page?: number };
296+
const page = body.page ?? 0;
297+
pagesRequested.push(page);
298+
299+
// Full first page, then a partial page: a correct iterator walks page 0 then page 1 and stops.
300+
const count = page === 0 ? hitsPerPage : 3;
301+
const hits = Array.from({ length: count }, (_, i) => ({ objectID: `page${page}-hit${i}` }));
302+
303+
return Promise.resolve({
304+
content: JSON.stringify({ hits, nbHits: hitsPerPage + 3 }),
305+
isTimedOut: false,
306+
status: 200,
307+
});
308+
},
309+
};
310+
311+
const paginatingClient = algoliasearch('APP_ID', 'API_KEY', { requester: paginatingRequester });
312+
313+
const objectIDs: string[] = [];
314+
await paginatingClient.browseSynonyms({
315+
indexName: 'my-index',
316+
aggregator: (response) => {
317+
for (const hit of response.hits) {
318+
objectIDs.push(hit.objectID);
319+
}
320+
},
321+
});
322+
323+
expect(pagesRequested).toEqual([0, 1]);
324+
expect(objectIDs).toHaveLength(hitsPerPage + 3);
325+
expect(new Set(objectIDs).size).toEqual(hitsPerPage + 3);
326+
});
327+
});
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
package com.algolia.client
2+
3+
import com.algolia.client.api.SearchClient
4+
import com.algolia.client.configuration.ClientOptions
5+
import com.algolia.client.extensions.browseSynonyms
6+
import com.algolia.client.model.search.SearchSynonymsParams
7+
import com.algolia.client.model.search.SynonymHit
8+
import io.ktor.client.engine.mock.MockEngine
9+
import io.ktor.client.engine.mock.respond
10+
import io.ktor.http.ContentType
11+
import io.ktor.http.HttpHeaders
12+
import io.ktor.http.HttpStatusCode
13+
import io.ktor.http.content.OutgoingContent
14+
import io.ktor.http.headersOf
15+
import kotlin.test.Test
16+
import kotlin.test.assertEquals
17+
import kotlin.test.assertTrue
18+
import kotlinx.coroutines.runBlocking
19+
import kotlinx.serialization.json.Json
20+
import kotlinx.serialization.json.int
21+
import kotlinx.serialization.json.jsonObject
22+
import kotlinx.serialization.json.jsonPrimitive
23+
24+
/**
25+
* Regression guard for the `browseSynonyms` pagination contract (Jira CR-11727).
26+
*
27+
* The Java `browseSynonyms` helper once failed to write the incremented `page` back into the
28+
* request params, so it re-fetched page 0 forever (infinite loop / OOM). Kotlin's implementation is
29+
* correct; this test locks that behavior in by mocking the HTTP layer and asserting that
30+
* consecutive, distinct pages are requested until a partial page ends the iteration.
31+
*
32+
* Placed under `jvmTest` (not `commonTest`) on purpose: the client module's test classpath has no
33+
* coroutine runner in `commonTest` (`kotlinx-coroutines-test` is not a declared dependency, and
34+
* `runBlocking` is not part of the JS-less multiplatform common source set), so a self-contained
35+
* coroutine-driven test must live on the JVM target where `runBlocking` is available. `ktor-client-mock`
36+
* is a `commonTest` dependency and is inherited here.
37+
*/
38+
class TestBrowseSynonyms {
39+
40+
private val hitsPerPage = 1000
41+
42+
/** Builds a canned `SearchSynonymsResponse` body with [count] hits prefixed by [objectIdPrefix]. */
43+
private fun synonymsPageBody(objectIdPrefix: String, count: Int): String {
44+
val hits =
45+
(0 until count).joinToString(",") { i ->
46+
"""{"objectID":"$objectIdPrefix-hit$i","type":"synonym"}"""
47+
}
48+
return """{"hits":[$hits],"nbHits":1003}"""
49+
}
50+
51+
@Test
52+
fun browseSynonymsPaginatesUntilPartialPage() = runBlocking {
53+
val requestedPages = mutableListOf<Int>()
54+
val requestedPaths = mutableListOf<String>()
55+
56+
val engine =
57+
MockEngine { request ->
58+
requestedPaths += request.url.encodedPath
59+
60+
val bodyBytes =
61+
(request.body as? OutgoingContent.ByteArrayContent)?.bytes()
62+
?: error("Unexpected request body type: ${request.body::class}")
63+
val page =
64+
Json.parseToJsonElement(bodyBytes.decodeToString())
65+
.jsonObject["page"]!!
66+
.jsonPrimitive
67+
.int
68+
requestedPages += page
69+
70+
// page 0 -> a full page (1000 hits) so iteration continues.
71+
// page >= 1 -> a partial page (3 hits) so iteration stops.
72+
val content =
73+
if (page == 0) synonymsPageBody("page0", hitsPerPage)
74+
else synonymsPageBody("page$page", 3)
75+
76+
respond(
77+
content = content,
78+
status = HttpStatusCode.OK,
79+
headers = headersOf(HttpHeaders.ContentType, ContentType.Application.Json.toString()),
80+
)
81+
}
82+
83+
val client =
84+
SearchClient(appId = "appId", apiKey = "apiKey", options = ClientOptions(engine = engine))
85+
86+
val collected = mutableListOf<SynonymHit>()
87+
client.use {
88+
it.browseSynonyms(
89+
indexName = "my-index",
90+
searchSynonymsParams = SearchSynonymsParams(),
91+
aggregator = { response -> collected += response.hits },
92+
)
93+
}
94+
95+
// The helper must request page 0 then page 1 (proving the incremented page is sent), and stop.
96+
assertEquals(listOf(0, 1), requestedPages, "browseSynonyms should request consecutive pages 0, 1")
97+
98+
// Every request must target the synonyms search endpoint (page read from that POST body).
99+
assertTrue(
100+
requestedPaths.all { it.endsWith("/synonyms/search") },
101+
"all requests should hit /synonyms/search, got $requestedPaths",
102+
)
103+
104+
// All hits across both pages are aggregated, with no page re-fetched (no duplicates).
105+
assertEquals(1003, collected.size, "aggregated hit count should be 1000 + 3")
106+
assertEquals(
107+
1003,
108+
collected.map { it.objectID }.toSet().size,
109+
"aggregated objectIDs should all be distinct (no page re-fetched)",
110+
)
111+
}
112+
}

‎clients/algoliasearch-client-php/lib/Iterators/RuleIterator.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ protected function fetchNextPage()
1515
{
1616
if (
1717
is_array($this->response)
18-
&& $this->key >= count($this->response['hits'])
18+
&& count($this->response['hits']) < $this->requestOptions['hitsPerPage']
1919
) {
2020
return;
2121
}

‎clients/algoliasearch-client-php/lib/Iterators/SynonymIterator.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ protected function fetchNextPage()
1515
{
1616
if (
1717
is_array($this->response)
18-
&& $this->key >= count($this->response['hits'])
18+
&& count($this->response['hits']) < $this->requestOptions['hitsPerPage']
1919
) {
2020
return;
2121
}

‎templates/java/api_helpers.mustache‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -501,14 +501,13 @@ public <T> Iterable<T> browseObjects(String indexName, Class<T> innerType) {
501501
*/
502502
public Iterable<SynonymHit> browseSynonyms(String indexName, SearchSynonymsParams params, RequestOptions requestOptions) {
503503
final Holder<Integer> currentPage = new Holder<>(0);
504-
505-
params.setPage(0);
506-
params.setHitsPerPage(1000);
504+
final int hitsPerPage = 1000;
505+
params.setHitsPerPage(hitsPerPage);
507506
508507
return AlgoliaIterableHelper.createIterable(
509508
() -> {
510-
SearchSynonymsResponse response = this.searchSynonyms(indexName, params, requestOptions);
511-
currentPage.value = response.getHits().size() < params.getHitsPerPage() ? null : currentPage.value + 1;
509+
SearchSynonymsResponse response = this.searchSynonyms(indexName, params.setPage(currentPage.value), requestOptions);
510+
currentPage.value = response.getHits().size() < hitsPerPage ? null : currentPage.value + 1;
512511
return response.getHits().iterator();
513512
},
514513
() -> currentPage.value != null
@@ -531,7 +530,7 @@ public Iterable<SynonymHit> browseSynonyms(String indexName, SearchSynonymsParam
531530
* @param indexName The index in which to perform the request.
532531
*/
533532
public Iterable<SynonymHit> browseSynonyms(String indexName) {
534-
return browseSynonyms(indexName, null, null);
533+
return browseSynonyms(indexName, new SearchSynonymsParams(), null);
535534
}
536535
537536
/**
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package com.algolia.manual;
2+
3+
import static org.junit.jupiter.api.Assertions.*;
4+
5+
import com.algolia.api.SearchClient;
6+
import com.algolia.config.ClientOptions;
7+
import com.algolia.config.HttpRequest;
8+
import com.algolia.config.RequestOptions;
9+
import com.algolia.config.Requester;
10+
import com.algolia.model.search.SearchSynonymsParams;
11+
import com.algolia.model.search.SearchSynonymsResponse;
12+
import com.algolia.model.search.SynonymHit;
13+
import com.fasterxml.jackson.core.type.TypeReference;
14+
import java.util.ArrayList;
15+
import java.util.Arrays;
16+
import java.util.List;
17+
import org.junit.jupiter.api.DisplayName;
18+
import org.junit.jupiter.api.Test;
19+
import org.junit.jupiter.api.Timeout;
20+
21+
/**
22+
* Regression test for CR-11727: `browseSynonyms` used to keep requesting `page=0` because the
23+
* incremented page was never written back to the params, so a full first page made the iterator
24+
* loop forever and duplicate hits. This test drives a full page followed by a partial page and
25+
* asserts the iterator walks page 0 then page 1 and stops.
26+
*/
27+
class BrowseSynonymsTest {
28+
29+
static final int HITS_PER_PAGE = 1000;
30+
31+
/**
32+
* Requester stub that returns a full page for `page=0` and a partial page afterwards, recording
33+
* the `page` sent on each `searchSynonyms` request. No real HTTP is performed.
34+
*/
35+
static class PaginatingRequester implements Requester {
36+
37+
final List<Integer> pagesRequested = new ArrayList<>();
38+
39+
private SearchSynonymsResponse handle(HttpRequest httpRequest) {
40+
SearchSynonymsParams params = (SearchSynonymsParams) httpRequest.getBody();
41+
int page = params.getPage() == null ? 0 : params.getPage();
42+
pagesRequested.add(page);
43+
44+
int count = page == 0 ? HITS_PER_PAGE : 3;
45+
List<SynonymHit> hits = new ArrayList<>();
46+
for (int i = 0; i < count; i++) {
47+
hits.add(new SynonymHit().setObjectID("page" + page + "-hit" + i));
48+
}
49+
50+
return new SearchSynonymsResponse().setHits(hits).setNbHits(HITS_PER_PAGE + 3);
51+
}
52+
53+
@Override
54+
@SuppressWarnings("unchecked")
55+
public <T> T execute(HttpRequest httpRequest, RequestOptions requestOptions, Class<?> returnType, Class<?> innerType) {
56+
return (T) handle(httpRequest);
57+
}
58+
59+
@Override
60+
@SuppressWarnings("unchecked")
61+
public <T> T execute(HttpRequest httpRequest, RequestOptions requestOptions, TypeReference<?> returnType) {
62+
return (T) handle(httpRequest);
63+
}
64+
65+
@Override
66+
public void close() {}
67+
}
68+
69+
@Test
70+
@Timeout(10)
71+
@DisplayName("browseSynonyms increments the page and terminates (CR-11727)")
72+
void browseSynonymsPaginatesAndStops() throws Exception {
73+
PaginatingRequester requester = new PaginatingRequester();
74+
75+
try (
76+
SearchClient client = new SearchClient("app-id", "api-key", ClientOptions.builder().setRequester(requester).build())
77+
) {
78+
List<SynonymHit> aggregated = new ArrayList<>();
79+
for (SynonymHit hit : client.browseSynonyms("my-index", new SearchSynonymsParams())) {
80+
aggregated.add(hit);
81+
// Fail fast instead of looping forever / OOMing if the page is never advanced.
82+
if (aggregated.size() > HITS_PER_PAGE + 100) {
83+
fail("browseSynonyms never advanced past page 0 and kept re-fetching the same page (CR-11727)");
84+
}
85+
}
86+
87+
assertEquals(Arrays.asList(0, 1), requester.pagesRequested, "browseSynonyms must request page 0 then page 1");
88+
assertEquals(HITS_PER_PAGE + 3, aggregated.size(), "browseSynonyms must aggregate every hit from both pages exactly once");
89+
90+
long distinctObjectIDs = aggregated.stream().map(SynonymHit::getObjectID).distinct().count();
91+
assertEquals(aggregated.size(), distinctObjectIDs, "browseSynonyms must not return duplicate hits (CR-11727)");
92+
}
93+
}
94+
}
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
<?php
2+
3+
namespace Algolia\AlgoliaSearch\Tests;
4+
5+
use Algolia\AlgoliaSearch\Api\SearchClient;
6+
use Algolia\AlgoliaSearch\Configuration\SearchConfig;
7+
use Algolia\AlgoliaSearch\Http\HttpClientInterface;
8+
use Algolia\AlgoliaSearch\Http\Psr7\Response;
9+
use Algolia\AlgoliaSearch\RequestOptions\RequestOptionsFactory;
10+
use Algolia\AlgoliaSearch\RetryStrategy\ApiWrapper;
11+
use Algolia\AlgoliaSearch\RetryStrategy\ClusterHosts;
12+
use PHPUnit\Framework\TestCase;
13+
use Psr\Http\Message\RequestInterface;
14+
15+
/**
16+
* Regression test for the `browseSynonyms` pagination: the page-based iterator must keep paginating
17+
* while a page is full and stop on the first non-full page. It previously stopped after the first
18+
* full page, silently truncating the results to `hitsPerPage`.
19+
*
20+
* @internal
21+
*
22+
* @coversNothing
23+
*/
24+
class BrowseSynonymsTest extends TestCase
25+
{
26+
public const HITS_PER_PAGE = 1000;
27+
28+
public function testBrowseSynonymsPaginatesAndStops(): void
29+
{
30+
$mockHttp = new class implements HttpClientInterface {
31+
/** @var int[] */
32+
public array $pagesRequested = [];
33+
34+
public function sendRequest(RequestInterface $request, $timeout, $connectTimeout)
35+
{
36+
$body = json_decode((string) $request->getBody(), true) ?: [];
37+
$page = $body['page'] ?? 0;
38+
$this->pagesRequested[] = $page;
39+
40+
// Full page for page 0, partial page afterwards, so a correct iterator walks page 0 then page 1.
41+
$count = 0 === $page ? BrowseSynonymsTest::HITS_PER_PAGE : 3;
42+
$hits = [];
43+
for ($i = 0; $i < $count; ++$i) {
44+
$hits[] = ['objectID' => "page{$page}-hit{$i}"];
45+
}
46+
47+
return new Response(
48+
200,
49+
['Content-Type' => 'application/json'],
50+
(string) json_encode(['hits' => $hits, 'nbHits' => BrowseSynonymsTest::HITS_PER_PAGE + 3])
51+
);
52+
}
53+
};
54+
55+
$config = SearchConfig::create('test-app-id', 'test-api-key')
56+
->setFullHosts(['http://localhost:9999'])
57+
;
58+
59+
$client = new SearchClient(
60+
new ApiWrapper($mockHttp, $config, ClusterHosts::create(['http://localhost:9999']), new RequestOptionsFactory($config)),
61+
$config
62+
);
63+
64+
$synonyms = [];
65+
foreach ($client->browseSynonyms('my-index') as $synonym) {
66+
$synonyms[] = $synonym;
67+
// Safety net so a regression can never hang the suite.
68+
if (count($synonyms) > self::HITS_PER_PAGE + 100) {
69+
$this->fail('browseSynonyms did not terminate');
70+
}
71+
}
72+
73+
$this->assertEquals([0, 1], $mockHttp->pagesRequested, 'browseSynonyms must request page 0 then page 1');
74+
$this->assertCount(self::HITS_PER_PAGE + 3, $synonyms, 'browseSynonyms must aggregate every hit from both pages');
75+
76+
$objectIDs = array_map(static fn ($hit) => $hit['objectID'], $synonyms);
77+
$this->assertCount(self::HITS_PER_PAGE + 3, array_unique($objectIDs), 'browseSynonyms must not return duplicate or truncated hits');
78+
}
79+
}

0 commit comments

Comments
 (0)