Skip to content

Commit d1f8a51

Browse files
authored
fix(javascript): chunked push polling offset and batchSize forwarding (#6598)
1 parent b1a7dd8 commit d1f8a51

4 files changed

Lines changed: 53 additions & 37 deletions

File tree

โ€Žscripts/cts/runCts.tsโ€Ž

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ export async function runCts(
190190
assertValidReplaceAllObjectsScopes(languages.length - skip('dart'));
191191
assertValidWaitForApiKey(languages.length - skip('dart'));
192192
assertPushMockValid(languages.length);
193-
assertValidChunkedPushWait(languages.length - skip('php') - skip('javascript'));
193+
assertValidChunkedPushWait(languages.length - skip('php'));
194194
}
195195
if (withBenchmarkServer) {
196196
printBenchmarkReport();

โ€Žtemplates/javascript/clients/algoliasearch/builds/definition.mustacheโ€Ž

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -129,23 +129,23 @@ export function algoliasearch(
129129
return {
130130
...client,
131131
132-
async saveObjectsWithTransformation({ indexName, objects, waitForTasks, maxRetries }, requestOptions): Promise<Array<WatchResponse>> {
132+
async saveObjectsWithTransformation({ indexName, objects, batchSize, waitForTasks, maxRetries }, requestOptions): Promise<Array<WatchResponse>> {
133133
if (!ingestionTransporter) {
134134
throw new Error('`transformationOptions` must be set in the client config before calling this method. It defaults to the Ingestion API defaults. See https://www.algolia.com/doc/libraries/sdk/methods/ingestion/');
135135
}
136136

137-
return ingestionTransporter.chunkedPush({ indexName, objects, action: 'addObject', waitForTasks, maxRetries }, requestOptions);
137+
return ingestionTransporter.chunkedPush({ indexName, objects, action: 'addObject', batchSize, waitForTasks, maxRetries }, requestOptions);
138138
},
139139

140140
async partialUpdateObjectsWithTransformation(
141-
{ indexName, objects, createIfNotExists, waitForTasks, maxRetries },
141+
{ indexName, objects, createIfNotExists, batchSize, waitForTasks, maxRetries },
142142
requestOptions,
143143
): Promise<Array<WatchResponse>> {
144144
if (!ingestionTransporter) {
145145
throw new Error('`transformationOptions` must be set in the client config before calling this method. It defaults to the Ingestion API defaults. See https://www.algolia.com/doc/libraries/sdk/methods/ingestion/');
146146
}
147147

148-
return ingestionTransporter.chunkedPush({ indexName, objects, action: createIfNotExists ? 'partialUpdateObject' : 'partialUpdateObjectNoCreate', waitForTasks, maxRetries }, requestOptions);
148+
return ingestionTransporter.chunkedPush({ indexName, objects, action: createIfNotExists ? 'partialUpdateObject' : 'partialUpdateObjectNoCreate', batchSize, waitForTasks, maxRetries }, requestOptions);
149149
},
150150

151151
async replaceAllObjectsWithTransformation(

โ€Žtemplates/javascript/clients/client/api/ingestionHelpers.mustacheโ€Ž

Lines changed: 47 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,49 @@ async chunkedPush(
2424
}: ChunkedPushOptions,
2525
requestOptions?: RequestOptions,
2626
): Promise<Array<WatchResponse>> {
27+
if (batchSize < 1) {
28+
throw new Error('`batchSize` must be at least 1.');
29+
}
30+
2731
let records: Array<PushTaskRecords> = [];
2832
let offset = 0;
2933
const responses: Array<WatchResponse> = [];
3034
const waitBatchSize = Math.floor(batchSize / 10) || batchSize;
3135

36+
// Polls every task in the `[start, end)` window of `responses` until it completes.
37+
const waitForBatch = async (start: number, end: number): Promise<void> => {
38+
for (const resp of responses.slice(start, end)) {
39+
if (!resp.eventID) {
40+
throw new Error('received unexpected response from the push endpoint, eventID must not be undefined');
41+
}
42+
43+
let retryCount = 0;
44+
45+
await createIterablePromise({
46+
func: async () => {
47+
if (resp.eventID === undefined || !resp.eventID) {
48+
throw new Error('received unexpected response from the push endpoint, eventID must not be undefined');
49+
}
50+
51+
return this.getEvent({ runID: resp.runID, eventID: resp.eventID }).catch((error: ApiError) => {
52+
if (error.status === 404) {
53+
return undefined;
54+
}
55+
56+
throw error;
57+
});
58+
},
59+
validate: (response) => response !== undefined,
60+
aggregator: () => (retryCount += 1),
61+
error: {
62+
validate: () => retryCount >= maxRetries,
63+
message: () => `Stopped waiting for the task after ${maxRetries} retries. This does not mean the operation failed; it may still complete. If you need to keep polling, retry with a higher maxRetries.`,
64+
},
65+
timeout: (): number => Math.min(retryCount * 1500, 5000),
66+
});
67+
}
68+
};
69+
3270
const objectEntries = objects.entries();
3371
for (const [i, obj] of objectEntries) {
3472
records.push(obj as PushTaskRecords);
@@ -39,40 +77,18 @@ async chunkedPush(
3977
records = [];
4078
}
4179

42-
if (waitForTasks && responses.length > 0 && (responses.length % waitBatchSize === 0 || i === objects.length - 1)) {
43-
for (const resp of responses.slice(offset, offset+waitBatchSize)) {
44-
if (!resp.eventID) {
45-
throw new Error('received unexpected response from the push endpoint, eventID must not be undefined');
46-
}
47-
48-
let retryCount = 0;
49-
50-
await createIterablePromise({
51-
func: async () => {
52-
if (resp.eventID === undefined || !resp.eventID) {
53-
throw new Error('received unexpected response from the push endpoint, eventID must not be undefined');
54-
}
55-
56-
return this.getEvent({ runID: resp.runID, eventID: resp.eventID }).catch((error: ApiError) => {
57-
if (error.status === 404) {
58-
return undefined;
59-
}
60-
61-
throw error;
62-
});
63-
},
64-
validate: (response) => response !== undefined,
65-
aggregator: () => (retryCount += 1),
66-
error: {
67-
validate: () => retryCount >= maxRetries,
68-
message: () => `Stopped waiting for the task after ${maxRetries} retries. This does not mean the operation failed; it may still complete. If you need to keep polling, retry with a higher maxRetries.`,
69-
},
70-
timeout: (): number => Math.min(retryCount * 1500, 5000),
71-
});
72-
}
80+
// Only wait once at least `waitBatchSize` un-polled responses have accumulated, and advance the
81+
// cursor by exactly that many, so every pushed task is polled once and only once.
82+
if (waitForTasks && responses.length - offset >= waitBatchSize) {
83+
await waitForBatch(offset, offset + waitBatchSize);
7384
offset += waitBatchSize;
7485
}
7586
}
7687

88+
// Drain the remaining responses that didn't fill a complete `waitBatchSize` window.
89+
if (waitForTasks) {
90+
await waitForBatch(offset, responses.length);
91+
}
92+
7793
return responses;
7894
},

โ€Žtests/CTS/client/search/saveObjectsWithTransformation.jsonโ€Ž

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
{
5858
"testName": "saveObjectsWithTransformation polls every task when waitForTasks is true",
5959
"autoCreateClient": false,
60-
"skipLanguages": ["php", "javascript"],
60+
"skipLanguages": ["php"],
6161
"steps": [
6262
{
6363
"type": "createClient",

0 commit comments

Comments
ย (0)