Skip to content

Commit b58c408

Browse files
authored
fix(specs): BREAKING CHANGE - make AB test pValue optional (#7066)
1 parent b1ad45d commit b58c408

15 files changed

Lines changed: 221 additions & 12 deletions

File tree

โ€Žclients/algoliasearch-client-javascript/package.jsonโ€Ž

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
"files": [
2828
{
2929
"path": "packages/algoliasearch/dist/algoliasearch.umd.js",
30-
"maxSize": "22KB"
30+
"maxSize": "22.25KB"
3131
},
3232
{
3333
"path": "packages/algoliasearch/dist/lite/builds/browser.umd.js",

โ€Žspecs/abtesting-v3/common/parameters.ymlโ€Ž

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,22 @@ ID:
77
schema:
88
$ref: '#/abTestID'
99

10+
Methods:
11+
in: query
12+
name: methods
13+
description: |
14+
Statistical analysis results to include, as a comma-separated list.
15+
When omitted, each test uses its configured method, or `frequentist` if no method is configured.
16+
Request both methods to include both sets of available results. This doesn't change the test configuration or compute missing results.
17+
Duplicate values aren't allowed.
18+
style: form
19+
explode: false
20+
schema:
21+
type: array
22+
minItems: 1
23+
items:
24+
$ref: 'schemas/ABTest.yml#/AnalysisMethod'
25+
1026
# misc
1127
index:
1228
type: string

โ€Žspecs/abtesting-v3/common/schemas/ABTest.ymlโ€Ž

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,10 +88,14 @@ ABTestConfiguration:
8888
$ref: '#/MetricsFilters'
8989
errorCorrection:
9090
$ref: '#/ErrorCorrectionType'
91+
method:
92+
$ref: '#/AnalysisMethod'
93+
primaryMetric:
94+
$ref: '#/PrimaryMetric'
9195

9296
MinimumDetectableEffect:
9397
type: object
94-
description: Configuration for the smallest difference between test variants you want to detect.
98+
description: Configuration for the smallest difference between test variants you want to detect, used to estimate the required sample size.
9599
properties:
96100
size:
97101
type: number
@@ -117,6 +121,20 @@ EffectMetric:
117121
- purchaseRate
118122
- noResultsRate
119123

124+
PrimaryMetric:
125+
type: string
126+
description: |
127+
Primary metric for Bayesian analysis. Required when `method` is `bayesian`.
128+
If the request includes a non-empty `metrics` list, this metric must be in that list.
129+
Revenue per search requires access to revenue analytics.
130+
example: conversion_rate
131+
enum:
132+
- add_to_cart_rate
133+
- click_through_rate
134+
- conversion_rate
135+
- purchase_rate
136+
- revenue_per_search
137+
120138
MetricsFilter:
121139
type: object
122140
additionalProperties: false
@@ -154,3 +172,12 @@ ErrorCorrectionType:
154172
enum:
155173
- bonferroni
156174
- benjamini-hochberg
175+
176+
AnalysisMethod:
177+
type: string
178+
description: |
179+
A/B test statistical analysis method.
180+
When omitted, the test is treated as `frequentist`. The server doesn't write a default value back to the configuration.
181+
enum:
182+
- bayesian
183+
- frequentist

โ€Žspecs/abtesting-v3/common/schemas/ABTestSettings.ymlโ€Ž

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ FeatureSettings:
66
name:
77
type: string
88
description: Name of the feature the settings belong to.
9-
example: 'reranking'
9+
example: 'drr'
1010
settings:
1111
description: Feature settings payload. The shape depends on the feature.
1212
required:

โ€Žspecs/abtesting-v3/common/schemas/Timeseries.ymlโ€Ž

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ MetricName:
4949
- user_count
5050
- tracked_user_count
5151
- no_result_count
52+
- tracked_no_result_count
5253
- add_to_cart_count
5354
- purchase_count
5455
- clicked_search_count
@@ -59,3 +60,5 @@ MetricName:
5960
- purchase_rate
6061
- average_click_position
6162
- revenue
63+
- revenue_per_search
64+
- average_order_value

โ€Žspecs/abtesting-v3/common/schemas/Variant.ymlโ€Ž

Lines changed: 75 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,14 @@ metricResult:
5353
properties:
5454
name:
5555
type: string
56+
description: Metric name. Revenue per search results use `revenue_per_search`.
5657
updatedAt:
5758
type: string
5859
description: Date and time when the metric was last updated, in RFC 3339 format.
5960
value:
6061
type: number
6162
format: double
63+
description: Metric value. For `revenue_per_search`, this is the winsorized mean revenue per search in the specified currency.
6264
valueCIHigh:
6365
type: number
6466
format: double
@@ -78,10 +80,12 @@ metricResult:
7880
pValue:
7981
type: number
8082
format: double
81-
description: PValue for the first variant (control) will always be 0. For the other variants, pValue is calculated for the current variant based on the control.
83+
description: |
84+
P-value for this variant compared to the control.
85+
Omitted when no p-value is available for this metric.
8286
dimension:
8387
type: string
84-
description: Dimension defined during test creation.
88+
description: Dimension defined during test creation. For revenue metrics, including `revenue_per_search`, this is the currency.
8589
metadata:
8690
$ref: '#/metricMetadata'
8791
criticalValue:
@@ -95,20 +99,21 @@ metricResult:
9599
type: boolean
96100
description: |
97101
Whether the pValue is significant or not based on the critical value and the error correction algorithm used.
102+
bayesian:
103+
$ref: '#/bayesianMetricResult'
98104
required:
99105
- name
100106
- updatedAt
101107
- value
102-
- pValue
103108
example:
104109
- name: addToCartCount
105110
updatedAt: 2025-06-15T15:06:44.400601Z
106111
value: 5
107-
pValue: 0.01
108112
- name: clickThroughRate
109113
updatedAt: 2025-05-15T17:52:15.644906Z
110114
value: 0.20869847452125934
111115
pValue: 0.004
116+
bayesian: {'probabilityToBeBetter': 0.991, 'evidence': {'status': 'enough'}}
112117
- name: revenue
113118
dimension: USD
114119
updatedAt: 2025-05-15T17:52:15.644906Z
@@ -121,6 +126,71 @@ metricResult:
121126
value: 999.66
122127
pValue: 0.04
123128
metadata: {'winsorizedValue': 888.8}
129+
- name: revenue_per_search
130+
dimension: USD
131+
updatedAt: 2025-05-15T17:52:15.644906Z
132+
value: 12.45
133+
metadata: {'mean': 12.45}
134+
bayesian:
135+
probabilityToBeBetter: 0.991
136+
relativeEffectCILow: 0.04
137+
relativeEffectCIHigh: 0.39
138+
evidence: {'status': 'unavailable'}
139+
140+
bayesianMetricResult:
141+
type: object
142+
description: |
143+
Bayesian inference results for this variant metric.
144+
Omitted when Bayesian results aren't requested or no Bayesian result is available for this metric.
145+
Individual inference fields can be omitted when their values aren't available.
146+
properties:
147+
probabilityToBeBetter:
148+
type: number
149+
format: double
150+
description: Probability that this variant is better than the control.
151+
example: 0.991
152+
relativeEffectCILow:
153+
type: number
154+
format: double
155+
description: Lower bound of the 95% credible interval for the relative effect (variant/control minus 1).
156+
example: 0.04
157+
relativeEffectCIHigh:
158+
type: number
159+
format: double
160+
description: Upper bound of the 95% credible interval for the relative effect (variant/control minus 1).
161+
example: 0.39
162+
evidence:
163+
$ref: '#/metricEvidence'
164+
165+
metricEvidence:
166+
type: object
167+
description: |
168+
Evidence status for this variant metric's comparison with the control.
169+
Only present when the API has an evidence status for the comparison. Omitted when evidence isn't applicable or no status exists.
170+
properties:
171+
status:
172+
$ref: '#/evidenceStatus'
173+
required:
174+
- status
175+
176+
evidenceStatus:
177+
type: string
178+
description: |
179+
Whether this variant metric's result is trustworthy yet.
180+
This doesn't mean the A/B test has ended, and it doesn't declare a winning variant.
181+
182+
- `enough`. Enough evidence has accumulated to trust this metric's result for this variant.
183+
- `not_enough`. Evidence was computed but hasn't reached the threshold yet.
184+
- `collecting`. Evidence is still accumulating, so it's too early to report a verdict.
185+
- `no_data`. No usable data for this variant metric, and none is expected for it.
186+
- `unavailable`. Evidence can't be computed for this variant metric's comparison.
187+
example: not_enough
188+
enum:
189+
- enough
190+
- not_enough
191+
- collecting
192+
- no_data
193+
- unavailable
124194

125195
variantMetadata:
126196
type: object
@@ -137,7 +207,7 @@ metricMetadata:
137207
type: number
138208
format: double
139209
description: |
140-
Only present in case the metric is 'revenue'.
210+
Only present for `revenue` metrics.
141211
It is the amount exceeding the 95th percentile of global revenue transactions involved in the AB Test. This amount is not considered when calculating statistical significance.
142212
It is tied to a per revenue-currency pair contrary to other
143213
global filter effects (such as outliers and empty search count).

โ€Žspecs/abtesting-v3/paths/abtest.ymlโ€Ž

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ get:
88
description: Retrieves the details for an A/B test by its ID.
99
parameters:
1010
- $ref: '../common/parameters.yml#/ID'
11+
- $ref: '../common/parameters.yml#/Methods'
1112
responses:
1213
'200':
1314
description: OK
@@ -30,6 +31,8 @@ get:
3031
$ref: '../../common/responses/MethodNotAllowed.yml'
3132
'404':
3233
$ref: '../../common/responses/IndexNotFound.yml'
34+
'422':
35+
$ref: '../../common/responses/UnprocessableEntity.yml'
3336

3437
delete:
3538
tags:

โ€Žspecs/abtesting-v3/paths/abtests.ymlโ€Ž

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ post:
6161
$ref: '../../common/responses/MethodNotAllowed.yml'
6262
'404':
6363
$ref: '../../common/responses/IndexNotFound.yml'
64+
'422':
65+
$ref: '../../common/responses/UnprocessableEntity.yml'
6466

6567
get:
6668
tags:
@@ -106,6 +108,7 @@ get:
106108
example: desc
107109
schema:
108110
$ref: '../common/parameters.yml#/direction'
111+
- $ref: '../common/parameters.yml#/Methods'
109112
responses:
110113
'200':
111114
description: OK
@@ -145,3 +148,5 @@ get:
145148
$ref: '../../common/responses/MethodNotAllowed.yml'
146149
'404':
147150
$ref: '../../common/responses/IndexNotFound.yml'
151+
'422':
152+
$ref: '../../common/responses/UnprocessableEntity.yml'

โ€Žspecs/abtesting-v3/paths/applyVariantSettings.ymlโ€Ž

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ post:
3636
'403':
3737
$ref: '../../common/responses/MethodNotAllowed.yml'
3838
'404':
39-
$ref: '../../common/responses/IndexNotFound.yml'
39+
description: A/B test, variant, or index not found.
40+
content:
41+
application/json:
42+
schema:
43+
$ref: '../../common/schemas/ErrorBase.yml'
4044
'422':
4145
$ref: '../../common/responses/UnprocessableEntity.yml'

โ€Žspecs/abtesting-v3/paths/timeseries.ymlโ€Ž

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ get:
1717
type: array
1818
items:
1919
$ref: '../common/schemas/Timeseries.yml#/MetricName'
20+
- $ref: '../common/parameters.yml#/Methods'
2021
responses:
2122
'200':
2223
description: OK

0 commit comments

Comments
ย (0)