Dataset Viewer
Duplicate
The dataset viewer is not available for this split.
Cannot load the dataset split (in streaming mode) to extract the first rows.
Error code:   StreamingRowsError
Exception:    CastError
Message:      Couldn't cast
edition: string
panel_id: string
index: double
raw_index: double
scores: struct<balanced_skill: double, balanced_raw: double, breadth_skill: double>
  child 0, balanced_skill: double
  child 1, balanced_raw: double
  child 2, breadth_skill: double
areas: list<item: struct<id: string, label: string, raw: double, skill: double, coverage: double, n: int64, (... 32 chars omitted)
  child 0, item: struct<id: string, label: string, raw: double, skill: double, coverage: double, n: int64, benchmarks (... 20 chars omitted)
      child 0, id: string
      child 1, label: string
      child 2, raw: double
      child 3, skill: double
      child 4, coverage: double
      child 5, n: int64
      child 6, benchmarks: list<item: int64>
          child 0, item: int64
benchmarks: struct<1: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_inde (... 4909 chars omitted)
  child 0, 1: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_index: bool, t (... 24 chars omitted)
      child 0, raw: double
      child 1, skill: double
      child 2, coverage: double
      child 3, random: double
      child 4, rule: string
      child 5, in_index: bool
      child 6, tracks: list<item: null>
          child 0, item: null
  child 1, 2: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_index: bool, t (... 24 chars omitted)
      child 0, raw: double
      child 1, skill: double
      child 2, covera
...
le
      child 3, random: double
      child 4, rule: string
      child 5, in_index: bool
  child 39, 24: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_index: bool>
      child 0, raw: double
      child 1, skill: double
      child 2, coverage: double
      child 3, random: double
      child 4, rule: string
      child 5, in_index: bool
  child 40, 26: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_index: bool>
      child 0, raw: double
      child 1, skill: double
      child 2, coverage: double
      child 3, random: double
      child 4, rule: string
      child 5, in_index: bool
  child 41, 27: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_index: bool>
      child 0, raw: double
      child 1, skill: double
      child 2, coverage: double
      child 3, random: double
      child 4, rule: string
      child 5, in_index: bool
  child 42, 34: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_index: bool>
      child 0, raw: double
      child 1, skill: double
      child 2, coverage: double
      child 3, random: double
      child 4, rule: string
      child 5, in_index: bool
coverage: double
note: string
counts: struct<ok: int64>
  child 0, ok: int64
successful_request_latency_ms: struct<median: double, p95: double, mean: double>
  child 0, median: double
  child 1, p95: double
  child 2, mean: double
engine: string
to
{'engine': Value('string'), 'counts': {'ok': Value('int64')}, 'successful_request_latency_ms': {'median': Value('float64'), 'p95': Value('float64'), 'mean': Value('float64')}, 'benchmarks': List({'catalog_id': Value('int64'), 'dataset': Value('string'), 'requests': Value('int64'), 'answered': Value('int64'), 'unsupported': Value('int64'), 'errors': Value('int64'), 'abstained': Value('int64'), 'pending': Value('int64'), 'scored_requests': Value('int64'), 'metric': Value('string'), 'score': Value('float64'), 'reference_same_cases': Value('null'), 'median_ms': Value('float64'), 'detail': {'field_accuracy': Value('float64'), 'case_exact_accuracy': Value('float64'), 'custom_metrics': {'ndcg_at_10': Value('float64'), 'mrr': Value('float64'), 'recall_at_10': Value('float64'), 'candidate_recall': Value('float64'), 'scorable_candidate_recall': Value('float64'), 'candidates_scored': Value('float64'), 'candidates_retrieved': Value('int64'), 'bm25_ndcg_at_10': Value('float64'), 'quality_quality': Value('float64'), 'quality_cost_usd': Value('float64'), 'quality_utility': Value('float64'), 'quality_oracle_optimal': Value('float64'), 'quality_utility_regret': Value('float64'), 'cost_aware_quality': Value('float64'), 'cost_aware_cost_usd': Value('float64'), 'cost_aware_utility': Value('float64'), 'cost_aware_oracle_optimal': Value('float64'), 'cost_aware_utility_regret': Value('float64'), 'value_regret': Value('float64'), 'normalized_rank_regret': Value('float64')}, 'positive_micro_f1': Valu
...
Value('int64'), 'mean': Value('int64')}}, 'cluster_macro_accuracy': Value('float64'), 'positive_f1_by_field': {'sarcastic': Value('float64'), 'sarcasm': Value('float64'), 'irony': Value('float64'), 'satire': Value('float64'), 'understatement': Value('float64'), 'overstatement': Value('float64'), 'rhetorical_question': Value('float64')}, 'category_macro_f1': Value('float64'), 'scored_fields': Value('int64'), 'chance_on_rows': Value('float64')}, 'tracks': {'RouterBench-0shot': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}, 'RouterBench-5shot': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}, 'iSarcasmEval-A-Ar': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-A-En': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-B-En': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-C-Ar': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-C-En': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'GSM8K-10choice': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}, 'GSM8K-4choice': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}}}), 'note': Value('string'), 'edition': Value('string')}
because column names don't match
Traceback:    Traceback (most recent call last):
                File "/src/services/worker/src/worker/utils.py", line 149, in get_rows_or_raise
                  return get_rows(
                      dataset=dataset,
                  ...<4 lines>...
                      column_names=column_names,
                  )
                File "/src/libs/libcommon/src/libcommon/utils.py", line 272, in decorator
                  return func(*args, **kwargs)
                File "/src/services/worker/src/worker/utils.py", line 129, in get_rows
                  rows_plus_one = list(itertools.islice(safe_iter(ds, dataset=dataset), rows_max_number + 1))
                File "/src/services/worker/src/worker/utils.py", line 544, in safe_iter
                  yield from ds.decode(False) if ds.features else ds
                File "/usr/local/lib/python3.14/site-packages/datasets/iterable_dataset.py", line 3052, in __iter__
                  for key, example in ex_iterable:
                                      ^^^^^^^^^^^
                File "/usr/local/lib/python3.14/site-packages/datasets/iterable_dataset.py", line 2559, in __iter__
                  for key, pa_table in self._iter_arrow():
                                       ~~~~~~~~~~~~~~~~^^
                File "/usr/local/lib/python3.14/site-packages/datasets/iterable_dataset.py", line 2584, in _iter_arrow
                  for key, pa_table in self.ex_iterable._iter_arrow():
                                       ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
                File "/usr/local/lib/python3.14/site-packages/datasets/iterable_dataset.py", line 558, in _iter_arrow
                  for key, pa_table in iterator:
                                       ^^^^^^^^
                File "/usr/local/lib/python3.14/site-packages/datasets/iterable_dataset.py", line 440, in _iter_arrow
                  for key, pa_table in self.generate_tables_fn(**gen_kwags):
                                       ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
                File "/usr/local/lib/python3.14/site-packages/datasets/packaged_modules/json/json.py", line 361, in _generate_tables
                  self._cast_table(pa_table, json_field_paths=json_field_paths),
                  ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
                File "/usr/local/lib/python3.14/site-packages/datasets/packaged_modules/json/json.py", line 132, in _cast_table
                  pa_table = table_cast(pa_table, self.info.features.arrow_schema)
                File "/usr/local/lib/python3.14/site-packages/datasets/table.py", line 2381, in table_cast
                  return cast_table_to_schema(table, schema)
                File "/usr/local/lib/python3.14/site-packages/datasets/table.py", line 2309, in cast_table_to_schema
                  raise CastError(
                  ...<3 lines>...
                  )
              datasets.table.CastError: Couldn't cast
              edition: string
              panel_id: string
              index: double
              raw_index: double
              scores: struct<balanced_skill: double, balanced_raw: double, breadth_skill: double>
                child 0, balanced_skill: double
                child 1, balanced_raw: double
                child 2, breadth_skill: double
              areas: list<item: struct<id: string, label: string, raw: double, skill: double, coverage: double, n: int64, (... 32 chars omitted)
                child 0, item: struct<id: string, label: string, raw: double, skill: double, coverage: double, n: int64, benchmarks (... 20 chars omitted)
                    child 0, id: string
                    child 1, label: string
                    child 2, raw: double
                    child 3, skill: double
                    child 4, coverage: double
                    child 5, n: int64
                    child 6, benchmarks: list<item: int64>
                        child 0, item: int64
              benchmarks: struct<1: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_inde (... 4909 chars omitted)
                child 0, 1: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_index: bool, t (... 24 chars omitted)
                    child 0, raw: double
                    child 1, skill: double
                    child 2, coverage: double
                    child 3, random: double
                    child 4, rule: string
                    child 5, in_index: bool
                    child 6, tracks: list<item: null>
                        child 0, item: null
                child 1, 2: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_index: bool, t (... 24 chars omitted)
                    child 0, raw: double
                    child 1, skill: double
                    child 2, covera
              ...
              le
                    child 3, random: double
                    child 4, rule: string
                    child 5, in_index: bool
                child 39, 24: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_index: bool>
                    child 0, raw: double
                    child 1, skill: double
                    child 2, coverage: double
                    child 3, random: double
                    child 4, rule: string
                    child 5, in_index: bool
                child 40, 26: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_index: bool>
                    child 0, raw: double
                    child 1, skill: double
                    child 2, coverage: double
                    child 3, random: double
                    child 4, rule: string
                    child 5, in_index: bool
                child 41, 27: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_index: bool>
                    child 0, raw: double
                    child 1, skill: double
                    child 2, coverage: double
                    child 3, random: double
                    child 4, rule: string
                    child 5, in_index: bool
                child 42, 34: struct<raw: double, skill: double, coverage: double, random: double, rule: string, in_index: bool>
                    child 0, raw: double
                    child 1, skill: double
                    child 2, coverage: double
                    child 3, random: double
                    child 4, rule: string
                    child 5, in_index: bool
              coverage: double
              note: string
              counts: struct<ok: int64>
                child 0, ok: int64
              successful_request_latency_ms: struct<median: double, p95: double, mean: double>
                child 0, median: double
                child 1, p95: double
                child 2, mean: double
              engine: string
              to
              {'engine': Value('string'), 'counts': {'ok': Value('int64')}, 'successful_request_latency_ms': {'median': Value('float64'), 'p95': Value('float64'), 'mean': Value('float64')}, 'benchmarks': List({'catalog_id': Value('int64'), 'dataset': Value('string'), 'requests': Value('int64'), 'answered': Value('int64'), 'unsupported': Value('int64'), 'errors': Value('int64'), 'abstained': Value('int64'), 'pending': Value('int64'), 'scored_requests': Value('int64'), 'metric': Value('string'), 'score': Value('float64'), 'reference_same_cases': Value('null'), 'median_ms': Value('float64'), 'detail': {'field_accuracy': Value('float64'), 'case_exact_accuracy': Value('float64'), 'custom_metrics': {'ndcg_at_10': Value('float64'), 'mrr': Value('float64'), 'recall_at_10': Value('float64'), 'candidate_recall': Value('float64'), 'scorable_candidate_recall': Value('float64'), 'candidates_scored': Value('float64'), 'candidates_retrieved': Value('int64'), 'bm25_ndcg_at_10': Value('float64'), 'quality_quality': Value('float64'), 'quality_cost_usd': Value('float64'), 'quality_utility': Value('float64'), 'quality_oracle_optimal': Value('float64'), 'quality_utility_regret': Value('float64'), 'cost_aware_quality': Value('float64'), 'cost_aware_cost_usd': Value('float64'), 'cost_aware_utility': Value('float64'), 'cost_aware_oracle_optimal': Value('float64'), 'cost_aware_utility_regret': Value('float64'), 'value_regret': Value('float64'), 'normalized_rank_regret': Value('float64')}, 'positive_micro_f1': Valu
              ...
              Value('int64'), 'mean': Value('int64')}}, 'cluster_macro_accuracy': Value('float64'), 'positive_f1_by_field': {'sarcastic': Value('float64'), 'sarcasm': Value('float64'), 'irony': Value('float64'), 'satire': Value('float64'), 'understatement': Value('float64'), 'overstatement': Value('float64'), 'rhetorical_question': Value('float64')}, 'category_macro_f1': Value('float64'), 'scored_fields': Value('int64'), 'chance_on_rows': Value('float64')}, 'tracks': {'RouterBench-0shot': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}, 'RouterBench-5shot': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}, 'iSarcasmEval-A-Ar': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-A-En': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-B-En': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-C-Ar': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'iSarcasmEval-C-En': {'metric': Value('string'), 'score': Value('null'), 'scored_requests': Value('int64')}, 'GSM8K-10choice': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}, 'GSM8K-4choice': {'metric': Value('string'), 'score': Value('float64'), 'scored_requests': Value('int64')}}}), 'note': Value('string'), 'edition': Value('string')}
              because column names don't match

Need help to make the dataset viewer work? Make sure to review how to configure the dataset viewer, and open a discussion for direct support.

H2O Lightning 4B on the Decision Index 0.3 public suite

Per-row results of H2O Lightning 4B v1.4.1 (commit cc18ee330e6895c41b9a60f37a0dd122dc4c3461) and v1.4.2 (the same weights; the third and fourth runs) on the Decision Index 0.3 public suite, text and vision, with the scores the Decision Index toolkit computes from them.

Run Rows Index
Text (43 benchmarks, the kit's 0.3 scorer) 140,178 scored 63.16 (raw 72.49)
Vision (11 benchmarks) 12,210 69.42
Text with the Decision Index profile (v1.4.2, PROFILE=di) 140,178 scored 63.64 (raw 72.81)
Vision at v1.4.2's image cap (1,638,400 pixels) 12,210 69.87

The first two runs are v1.4.1's, behind the model card's earlier Decision Index numbers. The model was served as the card's "For benchmark maintainers: Decision Index" section describes (two-way reading on one engine, one read per question, images and Box questions on), on vLLM 0.30.0 with NVIDIA L40S GPUs.

The third run is v1.4.2 (the same weights) served with its opt-in Decision Index profile, PROFILE=di bash h2o-lightning-4b/serve.sh with the card's text settings, on one NVIDIA RTX PRO 6000: a choice or score question whose top probability is under 0.8 is read a second time with the input written twice, and is answered with 2/3 of the first reading plus 1/3 of the second (effort_used.reread in each response). Read the same way without the profile, the same setup gives 63.19.

The fourth run is v1.4.2's vision serving: the package's serve.sh with no flags, images scaled to at most 1,638,400 pixels (v1.4.1: 1,048,576), two-way reading, on one NVIDIA RTX PRO 6000. It was read on a v1.4.2 release candidate with the same weights and image settings as the release; the release's later changes (the yes/no floor off by default, a client hang-up fix) do not change answers, which is what the vision index counts. The 69.42 run above stays as v1.4.1's.

Files

runs/h2o-lightning-4b/                 text run
  results.jsonl.gz                     one line per request: the request payload, the model's response, timings
  scores.json                          the kit's score output (index, areas, per-benchmark scores)
  benchmark-summary.json, index.json   the same, as the kit writes them
  shards/shardNN/environment.json      engine settings of each of the 16 shards
  shards/shardNN/status.json           completion record of each shard
runs/h2o-lightning-4b-vision/          vision run
  results.jsonl.gz, environment.json, status.json, vision-scores.json
runs/h2o-lightning-4b-di-profile/      text run of v1.4.2 with PROFILE=di (the same files as the text run)
runs/h2o-lightning-4b-v142-vision/     vision run of v1.4.2 (the same files as the vision run)

results.jsonl.gz of the text run has 140,620 lines; the kit's scorer leaves out the 442 requests the suite marks as not scoreable, which gives the 140,178 scored rows.

The request payloads are the suite's public benchmark texts, unchanged. One row of the BRIGHT retrieval benchmark (stackoverflow) quotes the AWS documentation's example access key (AKIA0123456787EXAMPL); it is a documentation placeholder, not a credential.

Rescore

With the Decision Index toolkit and its 0.3 suites:

gunzip -k runs/h2o-lightning-4b/results.jsonl.gz runs/h2o-lightning-4b-vision/results.jsonl.gz
decision-index score --edition 0.3 --suite-dir suite-0.3 --results runs/h2o-lightning-4b/results.jsonl --engine http --out text
decision-index vision score --dir vision --results runs/h2o-lightning-4b-vision/results.jsonl --out vision
gunzip -k runs/h2o-lightning-4b-di-profile/results.jsonl.gz
decision-index score --edition 0.3 --suite-dir suite-0.3 --results runs/h2o-lightning-4b-di-profile/results.jsonl --engine http --out text-di
gunzip -k runs/h2o-lightning-4b-v142-vision/results.jsonl.gz
decision-index vision score --dir vision --results runs/h2o-lightning-4b-v142-vision/results.jsonl --out vision-v142

This gives text index 63.16, vision index 69.42, with the profile text index 63.64, and for v1.4.2's vision run 69.87.

License

The results are released for evaluation and research. The request payloads inside results.jsonl.gz come from the benchmarks of the Decision Index public suite; each keeps the license of its original benchmark (see LICENSE.md). The model is licensed under Apache-2.0.

Downloads last month
250

Collection including h2oai/h2o-lightning-4b-decision-index