> ## Documentation Index > Fetch the complete documentation index at: https://openbench.dev/llms.txt > Use this file to discover all available pages before exploring further. # Benchmarks Catalog > Complete catalog of all available benchmarks with evaluation commands export const evalGroupsData = [{ "name": "AGIEval", "description": "Aggregate of 17 AGIEval exam tasks testing human-level reasoning across various domains", "category": "eval-group", "tags": ["eval-group"], "id": "agieval", "benchmark_count": 17, "benchmarks": ["agieval_aqua_rat", "agieval_gaokao_biology", "agieval_gaokao_chemistry", "agieval_gaokao_chinese", "agieval_gaokao_english", "agieval_gaokao_geography", "agieval_gaokao_history", "agieval_gaokao_mathqa", "agieval_gaokao_physics", "agieval_logiqa_en", "agieval_logiqa_zh", "agieval_lsat_ar", "agieval_lsat_lr", "agieval_lsat_rc", "agieval_sat_en", "agieval_sat_en_without_passage", "agieval_sat_math"] }, { "name": "ANLI", "description": "Aggregate of 3 ANLI rounds", "category": "eval-group", "tags": ["eval-group"], "id": "anli", "benchmark_count": 3, "benchmarks": ["anli_r1", "anli_r2", "anli_r3"] }, { "name": "Arabic Exams", "description": "Aggregate of 40+ Arabic exam tasks", "category": "eval-group", "tags": ["eval-group"], "id": "arabic_exams", "benchmark_count": 41, "benchmarks": ["arabic_exams_accounting_university", "arabic_exams_arabic_language_general", "arabic_exams_arabic_language_grammar", "arabic_exams_arabic_language_high_school", "arabic_exams_arabic_language_middle_school", "arabic_exams_arabic_language_primary_school", "arabic_exams_biology_high_school", "arabic_exams_civics_high_school", "arabic_exams_civics_middle_school", "arabic_exams_computer_science_high_school", "arabic_exams_computer_science_middle_school", "arabic_exams_computer_science_primary_school", "arabic_exams_computer_science_university", "arabic_exams_driving_test", "arabic_exams_economics_high_school", "arabic_exams_economics_middle_school", "arabic_exams_economics_university", "arabic_exams_general_knowledge", "arabic_exams_general_knowledge_middle_school", "arabic_exams_general_knowledge_primary_school", "arabic_exams_geography_high_school", "arabic_exams_geography_middle_school", "arabic_exams_geography_primary_school", "arabic_exams_history_high_school", "arabic_exams_history_middle_school", "arabic_exams_history_primary_school", "arabic_exams_islamic_studies_general", "arabic_exams_islamic_studies_high_school", "arabic_exams_islamic_studies_middle_school", "arabic_exams_islamic_studies_primary_school", "arabic_exams_law_professional", "arabic_exams_management_university", "arabic_exams_math_high_school", "arabic_exams_math_primary_school", "arabic_exams_natural_science_middle_school", "arabic_exams_natural_science_primary_school", "arabic_exams_philosophy_high_school", "arabic_exams_physics_high_school", "arabic_exams_political_science_university", "arabic_exams_social_science_middle_school", "arabic_exams_social_science_primary_school"] }, { "name": "BBQ (Bias Benchmark for QA)", "description": "Aggregate of 11 BBQ bias evaluation tasks across demographic categories", "category": "eval-group", "tags": ["eval-group"], "id": "bbq", "benchmark_count": 11, "benchmarks": ["bbq_age", "bbq_disability_status", "bbq_gender_identity", "bbq_nationality", "bbq_physical_appearance", "bbq_race_ethnicity", "bbq_race_x_ses", "bbq_race_x_gender", "bbq_religion", "bbq_ses", "bbq_sexual_orientation"] }, { "name": "BIG-Bench", "description": "Aggregate of 121 BIG-Bench tasks for comprehensive language model evaluation", "category": "eval-group", "tags": ["eval-group"], "id": "bigbench", "benchmark_count": 121, "benchmarks": ["bigbench_anachronisms", "bigbench_analogical_similarity", "bigbench_analytic_entailment", "bigbench_arithmetic", "bigbench_authorship_verification", "bigbench_bbq_lite_json", "bigbench_causal_judgment", "bigbench_cause_and_effect", "bigbench_checkmate_in_one", "bigbench_cifar10_classification", "bigbench_code_line_description", "bigbench_color", "bigbench_common_morpheme", "bigbench_conceptual_combinations", "bigbench_contextual_parametric_knowledge_conflicts", "bigbench_crash_blossom", "bigbench_crass_ai", "bigbench_cryobiology_spanish", "bigbench_cs_algorithms", "bigbench_dark_humor_detection", "bigbench_date_understanding", "bigbench_disambiguation_qa", "bigbench_discourse_marker_prediction", "bigbench_dyck_languages", "bigbench_elementary_math_qa", "bigbench_emoji_movie", "bigbench_emojis_emotion_prediction", "bigbench_empirical_judgments", "bigbench_english_proverbs", "bigbench_english_russian_proverbs", "bigbench_entailed_polarity", "bigbench_entailed_polarity_hindi", "bigbench_epistemic_reasoning", "bigbench_evaluating_information_essentiality", "bigbench_fact_checker", "bigbench_fantasy_reasoning", "bigbench_figure_of_speech_detection", "bigbench_formal_fallacies_syllogisms_negation", "bigbench_general_knowledge", "bigbench_geometric_shapes", "bigbench_goal_step_wikihow", "bigbench_gre_reading_comprehension", "bigbench_hhh_alignment", "bigbench_hindu_knowledge", "bigbench_hinglish_toxicity", "bigbench_human_organs_senses", "bigbench_hyperbaton", "bigbench_identify_math_theorems", "bigbench_identify_odd_metaphor", "bigbench_implicatures", "bigbench_implicit_relations", "bigbench_indic_cause_and_effect", "bigbench_intent_recognition", "bigbench_international_phonetic_alphabet_nli", "bigbench_intersect_geometry", "bigbench_irony_identification", "bigbench_kanji_ascii", "bigbench_kannada", "bigbench_key_value_maps", "bigbench_known_unknowns", "bigbench_language_identification", "bigbench_logic_grid_puzzle", "bigbench_logical_args", "bigbench_logical_deduction", "bigbench_logical_fallacy_detection", "bigbench_logical_sequence", "bigbench_mathematical_induction", "bigbench_medical_questions_russian", "bigbench_metaphor_boolean", "bigbench_metaphor_understanding", "bigbench_minute_mysteries_qa", "bigbench_misconceptions", "bigbench_misconceptions_russian", "bigbench_mnist_ascii", "bigbench_moral_permissibility", "bigbench_movie_dialog_same_or_different", "bigbench_movie_recommendation", "bigbench_navigate", "bigbench_nonsense_words_grammar", "bigbench_novel_concepts", "bigbench_odd_one_out", "bigbench_parsinlu_qa", "bigbench_penguins_in_a_table", "bigbench_periodic_elements", "bigbench_persian_idioms", "bigbench_phrase_relatedness", "bigbench_physical_intuition", "bigbench_physics", "bigbench_play_dialog_same_or_different", "bigbench_presuppositions_as_nli", "bigbench_question_selection", "bigbench_real_or_fake_text", "bigbench_reasoning_about_colored_objects", "bigbench_rhyming", "bigbench_riddle_sense", "bigbench_ruin_names", "bigbench_salient_translation_error_detection", "bigbench_sentence_ambiguity", "bigbench_similarities_abstraction", "bigbench_simple_ethical_questions", "bigbench_snarks", "bigbench_social_iqa", "bigbench_social_support", "bigbench_sports_understanding", "bigbench_strange_stories", "bigbench_strategyqa", "bigbench_suicide_risk", "bigbench_swahili_english_proverbs", "bigbench_swedish_to_german_proverbs", "bigbench_symbol_interpretation", "bigbench_temporal_sequences", "bigbench_timedial", "bigbench_tracking_shuffled_objects", "bigbench_understanding_fables", "bigbench_undo_permutation", "bigbench_unit_conversion", "bigbench_unit_interpretation", "bigbench_vitaminc_fact_verification", "bigbench_what_is_the_tao", "bigbench_which_wiki_edit", "bigbench_winowhy"] }, { "name": "BIG-Bench Hard", "description": "Aggregate of 18 challenging BIG-Bench tasks that require multi-step reasoning", "category": "eval-group", "tags": ["eval-group"], "id": "bbh", "benchmark_count": 18, "benchmarks": ["bbh_causal_judgment", "bbh_date_understanding", "bbh_disambiguation_qa", "bbh_geometric_shapes", "bbh_logical_deduction_five_objects", "bbh_logical_deduction_seven_objects", "bbh_logical_deduction_three_objects", "bbh_movie_recommendation", "bbh_navigate", "bbh_reasoning_about_colored_objects", "bbh_ruin_names", "bbh_salient_translation_error_detection", "bbh_snarks", "bbh_sports_understanding", "bbh_temporal_sequences", "bbh_tracking_shuffled_objects_five_objects", "bbh_tracking_shuffled_objects_seven_objects", "bbh_tracking_shuffled_objects_three_objects"] }, { "name": "BIG-Bench Lite", "description": "BIG-Bench Lite - 18 selected BBH tasks", "category": "eval-group", "tags": ["eval-group"], "id": "bbl", "benchmark_count": 18, "benchmarks": ["bbh_causal_judgment", "bbh_date_understanding", "bbh_disambiguation_qa", "bbh_geometric_shapes", "bbh_logical_deduction_five_objects", "bbh_logical_deduction_seven_objects", "bbh_logical_deduction_three_objects", "bbh_movie_recommendation", "bbh_navigate", "bbh_reasoning_about_colored_objects", "bbh_ruin_names", "bbh_salient_translation_error_detection", "bbh_snarks", "bbh_sports_understanding", "bbh_temporal_sequences", "bbh_tracking_shuffled_objects_five_objects", "bbh_tracking_shuffled_objects_seven_objects", "bbh_tracking_shuffled_objects_three_objects"] }, { "name": "BLiMP", "description": "Aggregate of 67 BLiMP linguistic tasks", "category": "eval-group", "tags": ["eval-group"], "id": "blimp", "benchmark_count": 67, "benchmarks": ["blimp_adjunct_island", "blimp_anaphor_gender_agreement", "blimp_anaphor_number_agreement", "blimp_animate_subject_passive", "blimp_animate_subject_trans", "blimp_causative", "blimp_complex_NP_island", "blimp_coordinate_structure_constraint_complex_left_branch", "blimp_coordinate_structure_constraint_object_extraction", "blimp_determiner_noun_agreement_1", "blimp_determiner_noun_agreement_2", "blimp_determiner_noun_agreement_irregular_1", "blimp_determiner_noun_agreement_irregular_2", "blimp_determiner_noun_agreement_with_adj_2", "blimp_determiner_noun_agreement_with_adj_irregular_1", "blimp_determiner_noun_agreement_with_adj_irregular_2", "blimp_determiner_noun_agreement_with_adjective_1", "blimp_distractor_agreement_relational_noun", "blimp_distractor_agreement_relative_clause", "blimp_drop_argument", "blimp_ellipsis_n_bar_1", "blimp_ellipsis_n_bar_2", "blimp_existential_there_object_raising", "blimp_existential_there_quantifiers_1", "blimp_existential_there_quantifiers_2", "blimp_existential_there_subject_raising", "blimp_expletive_it_object_raising", "blimp_inchoative", "blimp_intransitive", "blimp_irregular_past_participle_adjectives", "blimp_irregular_past_participle_verbs", "blimp_irregular_plural_subject_verb_agreement_1", "blimp_irregular_plural_subject_verb_agreement_2", "blimp_left_branch_island_echo_question", "blimp_left_branch_island_simple_question", "blimp_matrix_question_npi_licensor_present", "blimp_npi_present_1", "blimp_npi_present_2", "blimp_only_npi_licensor_present", "blimp_only_npi_scope", "blimp_passive_1", "blimp_passive_2", "blimp_principle_A_c_command", "blimp_principle_A_case_1", "blimp_principle_A_case_2", "blimp_principle_A_domain_1", "blimp_principle_A_domain_2", "blimp_principle_A_domain_3", "blimp_principle_A_reconstruction", "blimp_regular_plural_subject_verb_agreement_1", "blimp_regular_plural_subject_verb_agreement_2", "blimp_sentential_negation_npi_licensor_present", "blimp_sentential_negation_npi_scope", "blimp_sentential_subject_island", "blimp_superlative_quantifiers_1", "blimp_superlative_quantifiers_2", "blimp_tough_vs_raising_1", "blimp_tough_vs_raising_2", "blimp_transitive", "blimp_wh_island", "blimp_wh_questions_object_gap", "blimp_wh_questions_subject_gap", "blimp_wh_questions_subject_gap_long_distance", "blimp_wh_vs_that_no_gap", "blimp_wh_vs_that_no_gap_long_distance", "blimp_wh_vs_that_with_gap", "blimp_wh_vs_that_with_gap_long_distance"] }, { "name": "CTI-Bench", "description": "Aggregate of 4 CTI-Bench tasks", "category": "eval-group", "tags": ["eval-group"], "id": "cti_bench", "benchmark_count": 4, "benchmarks": ["cti_bench_ate", "cti_bench_mcq", "cti_bench_rcm", "cti_bench_vsp"] }, { "name": "ETHICS", "description": "Aggregate of 5 ETHICS moral reasoning tasks", "category": "eval-group", "tags": ["eval-group"], "id": "ethics", "benchmark_count": 5, "benchmarks": ["ethics_commonsense", "ethics_deontology", "ethics_justice", "ethics_utilitarianism", "ethics_virtue"] }, { "name": "GLUE", "description": "Aggregate of 10 GLUE NLU tasks", "category": "eval-group", "tags": ["eval-group"], "id": "glue", "benchmark_count": 10, "benchmarks": ["glue_cola", "glue_mnli", "glue_mnli_mismatched", "glue_mrpc", "glue_qnli", "glue_qqp", "glue_rte", "glue_sst2", "glue_stsb", "glue_wnli"] }, { "name": "Global-MMLU", "description": "Aggregate of 42 Global-MMLU language tasks", "category": "eval-group", "tags": ["eval-group"], "id": "global_mmlu", "benchmark_count": 42, "benchmarks": ["global_mmlu_amharic", "global_mmlu_arabic", "global_mmlu_bengali", "global_mmlu_chichewa", "global_mmlu_chinese", "global_mmlu_czech", "global_mmlu_dutch", "global_mmlu_english", "global_mmlu_filipino", "global_mmlu_french", "global_mmlu_german", "global_mmlu_greek", "global_mmlu_hausa", "global_mmlu_hebrew", "global_mmlu_hindi", "global_mmlu_igbo", "global_mmlu_indonesian", "global_mmlu_italian", "global_mmlu_japanese", "global_mmlu_korean", "global_mmlu_kyrgyz", "global_mmlu_lithuanian", "global_mmlu_malagasy", "global_mmlu_malay", "global_mmlu_nepali", "global_mmlu_persian", "global_mmlu_polish", "global_mmlu_portuguese", "global_mmlu_romanian", "global_mmlu_russian", "global_mmlu_serbian", "global_mmlu_shona", "global_mmlu_sinhala", "global_mmlu_somali", "global_mmlu_spanish", "global_mmlu_swahili", "global_mmlu_swedish", "global_mmlu_telugu", "global_mmlu_turkish", "global_mmlu_ukrainian", "global_mmlu_vietnamese", "global_mmlu_yoruba"] }, { "name": "HealthBench", "description": "Aggregate of HealthBench tasks", "category": "eval-group", "tags": ["eval-group"], "id": "healthbench", "benchmark_count": 2, "benchmarks": ["healthbench_consensus", "healthbench_hard"] }, { "name": "M2S conversions of multi-turn jailbreak datasets", "description": "Aggregate of 3 M2S conversions of multi-turn jailbreak datasets", "category": "eval-group", "tags": ["eval-group"], "id": "jailbreak", "benchmark_count": 3, "benchmarks": ["safemt_m2s", "cosafe_m2s", "mhj_m2s"] }, { "name": "MGSM", "description": "Multilingual Grade School Math across 11 languages", "category": "eval-group", "tags": ["eval-group"], "id": "mgsm", "benchmark_count": 13, "benchmarks": ["mgsm_en", "mgsm_de", "mgsm_es", "mgsm_fr", "mgsm_sw", "mgsm_bn", "mgsm_ja", "mgsm_ru", "mgsm_te", "mgsm_th", "mgsm_zh", "mgsm_latin", "mgsm_non_latin"] }, { "name": "MMMLU", "description": "MMLU in 14 languages", "category": "eval-group", "tags": ["eval-group"], "id": "mmmlu", "benchmark_count": 14, "benchmarks": ["mmmlu_ar_xy", "mmmlu_bn_bd", "mmmlu_de_de", "mmmlu_es_la", "mmmlu_fr_fr", "mmmlu_hi_in", "mmmlu_id_id", "mmmlu_it_it", "mmmlu_ja_jp", "mmmlu_ko_kr", "mmmlu_pt_br", "mmmlu_zh_cn", "mmmlu_sw_ke", "mmmlu_yo_ng"] }, { "name": "MathArena", "description": "Aggregate of 11 math competition tasks", "category": "eval-group", "tags": ["eval-group"], "id": "matharena", "benchmark_count": 11, "benchmarks": ["aime_2023_I", "aime_2023_II", "aime_2024", "aime_2024_I", "aime_2024_II", "aime_2025", "aime_2025_II", "brumo_2025", "hmmt_feb_2023", "hmmt_feb_2024", "hmmt_feb_2025"] }, { "name": "Mathematics - 22 Oct 2025", "description": "Comprehensive mathematics evaluation including competition math, multilingual grade school math, and advanced problem solving", "category": "eval-group", "tags": ["eval-group"], "id": "math_group_22_10_2025", "benchmark_count": 11, "benchmarks": ["aime_2025", "aime_2025_II", "hmmt_feb_2025", "brumo_2025", "otis_mock_aime_2025", "smt_algebra", "smt_calculus", "smt_discrete", "smt_general", "smt_geometry", "smt_guts"] }, { "name": "Multimodal Benchmarks", "description": "Select suite of vision-language benchmarks (note: this is not a comprehensive list of all multimodal benchmarks)", "category": "eval-group", "tags": ["eval-group"], "id": "multimodal", "benchmark_count": 7, "benchmarks": ["chartqapro", "mathvista", "mmmu_pro_vision", "mmstar", "mmvetv2", "ocrbenchv2", "rocketscience"] }, { "name": "OTIS Mock AIME", "description": "Aggregate of 2 Mock AIME years", "category": "eval-group", "tags": ["eval-group"], "id": "otis_mock_aime", "benchmark_count": 2, "benchmarks": ["otis_mock_aime_2024", "otis_mock_aime_2025"] }, { "name": "OpenAI MRCR", "description": "Aggregate of 3 MRCR needle tasks", "category": "eval-group", "tags": ["eval-group"], "id": "openai_mrcr", "benchmark_count": 3, "benchmarks": ["openai_mrcr_2n", "openai_mrcr_4n", "openai_mrcr_8n"] }, { "name": "PolygloToxicityPrompts", "description": "Multilingual evaluation of neural toxic degeneration across 17 languages", "category": "eval-group", "tags": ["eval-group"], "id": "polyglotoxicity", "benchmark_count": 17, "benchmarks": ["polyglotoxicity_arabic", "polyglotoxicity_chinese", "polyglotoxicity_czech", "polyglotoxicity_dutch", "polyglotoxicity_english", "polyglotoxicity_french", "polyglotoxicity_german", "polyglotoxicity_hindi", "polyglotoxicity_indonesian", "polyglotoxicity_italian", "polyglotoxicity_japanese", "polyglotoxicity_korean", "polyglotoxicity_polish", "polyglotoxicity_portuguese", "polyglotoxicity_russian", "polyglotoxicity_swedish", "polyglotoxicity_spanish"] }, { "name": "QA4MRE", "description": "Aggregate of 3 QA4MRE years", "category": "eval-group", "tags": ["eval-group"], "id": "qa4mre", "benchmark_count": 3, "benchmarks": ["qa4mre_2011", "qa4mre_2012", "qa4mre_2013"] }, { "name": "SMT 2024", "description": "Aggregate of 6 Stanford Math Tournament 2024 categories", "category": "eval-group", "tags": ["eval-group"], "id": "smt", "benchmark_count": 6, "benchmarks": ["smt_algebra", "smt_calculus", "smt_discrete", "smt_general", "smt_geometry", "smt_guts"] }, { "name": "SuperGLUE", "description": "SuperGLUE benchmark with 6 NLU tasks", "category": "eval-group", "tags": ["eval-group"], "id": "superglue", "benchmark_count": 7, "benchmarks": ["boolq", "cb", "copa", "multirc", "rte", "wic", "wsc"] }, { "name": "TauBench", "description": "Aggregate of 3 TauBench tasks", "category": "eval-group", "tags": ["eval-group"], "id": "tau_bench", "benchmark_count": 3, "benchmarks": ["tau_bench_retail", "tau_bench_airline", "tau_bench_telecom"] }, { "name": "XCOPA", "description": "Aggregate of 11 XCOPA language tasks", "category": "eval-group", "tags": ["eval-group"], "id": "xcopa", "benchmark_count": 11, "benchmarks": ["xcopa_et", "xcopa_ht", "xcopa_id", "xcopa_it", "xcopa_qu", "xcopa_sw", "xcopa_ta", "xcopa_th", "xcopa_tr", "xcopa_vi", "xcopa_zh"] }, { "name": "XStoryCloze", "description": "Aggregate of 11 XStoryCloze tasks", "category": "eval-group", "tags": ["eval-group"], "id": "xstorycloze", "benchmark_count": 11, "benchmarks": ["xstorycloze_ar", "xstorycloze_en", "xstorycloze_es", "xstorycloze_eu", "xstorycloze_hi", "xstorycloze_id", "xstorycloze_my", "xstorycloze_ru", "xstorycloze_sw", "xstorycloze_te", "xstorycloze_zh"] }, { "name": "XWinograd", "description": "Aggregate of 6 XWinograd tasks", "category": "eval-group", "tags": ["eval-group"], "id": "xwinograd", "benchmark_count": 6, "benchmarks": ["xwinograd_en", "xwinograd_fr", "xwinograd_jp", "xwinograd_pt", "xwinograd_ru", "xwinograd_zh"] }]; export const benchmarksData = [{ "name": "AGIEval (All Subsets)", "description": "Human-centric benchmark with 17 official qualifying exam questions testing general cognitive abilities", "category": "agieval", "tags": ["multiple-choice", "academic-exams", "reasoning", "cognitive-abilities"], "function_name": "agieval", "is_alpha": false }, { "name": "AGIEval: AQUA-RAT", "description": "Algebraic question answering and reasoning", "category": "agieval", "tags": ["multiple-choice", "algebra", "reasoning", "math", "agieval"], "function_name": "agieval_aqua_rat", "is_alpha": false }, { "name": "AGIEval: Gaokao Biology", "description": "Chinese national college entrance exam - Biology", "category": "agieval", "tags": ["multiple-choice", "gaokao", "biology", "science", "agieval"], "function_name": "agieval_gaokao_biology", "is_alpha": false }, { "name": "AGIEval: Gaokao Chemistry", "description": "Chinese national college entrance exam - Chemistry", "category": "agieval", "tags": ["multiple-choice", "gaokao", "chemistry", "science", "agieval"], "function_name": "agieval_gaokao_chemistry", "is_alpha": false }, { "name": "AGIEval: Gaokao Chinese", "description": "Chinese national college entrance exam - Chinese language", "category": "agieval", "tags": ["multiple-choice", "gaokao", "chinese", "language", "agieval"], "function_name": "agieval_gaokao_chinese", "is_alpha": false }, { "name": "AGIEval: Gaokao English", "description": "Chinese national college entrance exam - English", "category": "agieval", "tags": ["multiple-choice", "gaokao", "english", "language", "agieval"], "function_name": "agieval_gaokao_english", "is_alpha": false }, { "name": "AGIEval: Gaokao Geography", "description": "Chinese national college entrance exam - Geography", "category": "agieval", "tags": ["multiple-choice", "gaokao", "geography", "social-studies", "agieval"], "function_name": "agieval_gaokao_geography", "is_alpha": false }, { "name": "AGIEval: Gaokao History", "description": "Chinese national college entrance exam - History", "category": "agieval", "tags": ["multiple-choice", "gaokao", "history", "social-studies", "agieval"], "function_name": "agieval_gaokao_history", "is_alpha": false }, { "name": "AGIEval: Gaokao Math", "description": "Chinese national college entrance exam - Mathematics", "category": "agieval", "tags": ["multiple-choice", "gaokao", "mathematics", "problem-solving", "agieval"], "function_name": "agieval_gaokao_mathqa", "is_alpha": false }, { "name": "AGIEval: Gaokao Physics", "description": "Chinese national college entrance exam - Physics", "category": "agieval", "tags": ["multiple-choice", "gaokao", "physics", "science", "agieval"], "function_name": "agieval_gaokao_physics", "is_alpha": false }, { "name": "AGIEval: LSAT Analytical Reasoning", "description": "Law School Admission Test - Analytical Reasoning section", "category": "agieval", "tags": ["multiple-choice", "law", "analytical-reasoning", "lsat", "agieval"], "function_name": "agieval_lsat_ar", "is_alpha": false }, { "name": "AGIEval: LSAT Logical Reasoning", "description": "Law School Admission Test - Logical Reasoning section", "category": "agieval", "tags": ["multiple-choice", "law", "logical-reasoning", "lsat", "agieval"], "function_name": "agieval_lsat_lr", "is_alpha": false }, { "name": "AGIEval: LSAT Reading Comprehension", "description": "Law School Admission Test - Reading Comprehension section", "category": "agieval", "tags": ["multiple-choice", "law", "reading-comprehension", "lsat", "agieval"], "function_name": "agieval_lsat_rc", "is_alpha": false }, { "name": "AGIEval: LogiQA (Chinese)", "description": "Logical reasoning questions in Chinese", "category": "agieval", "tags": ["multiple-choice", "logic", "reasoning", "chinese", "agieval"], "function_name": "agieval_logiqa_zh", "is_alpha": false }, { "name": "AGIEval: LogiQA (English)", "description": "Logical reasoning questions in English", "category": "agieval", "tags": ["multiple-choice", "logic", "reasoning", "english", "agieval"], "function_name": "agieval_logiqa_en", "is_alpha": false }, { "name": "AGIEval: SAT English", "description": "Scholastic Assessment Test - English section", "category": "agieval", "tags": ["multiple-choice", "sat", "english", "reading", "agieval"], "function_name": "agieval_sat_en", "is_alpha": false }, { "name": "AGIEval: SAT English (No Passage)", "description": "SAT English questions without reading passages", "category": "agieval", "tags": ["multiple-choice", "sat", "english", "grammar", "agieval"], "function_name": "agieval_sat_en_without_passage", "is_alpha": false }, { "name": "AGIEval: SAT Math", "description": "Scholastic Assessment Test - Math section", "category": "agieval", "tags": ["multiple-choice", "sat", "mathematics", "problem-solving", "agieval"], "function_name": "agieval_sat_math", "is_alpha": false }, { "name": "AIME 2023 I", "description": "American Invitational Mathematics Examination 2023 (First)", "category": "math", "tags": ["math", "competition", "aime", "2023"], "function_name": "aime_2023_I", "is_alpha": false }, { "name": "AIME 2023 II", "description": "American Invitational Mathematics Examination 2023 (Second)", "category": "math", "tags": ["math", "competition", "aime", "2023"], "function_name": "aime_2023_II", "is_alpha": false }, { "name": "AIME 2024", "description": "American Invitational Mathematics Examination 2024 (Combined I & II)", "category": "math", "tags": ["math", "competition", "aime", "2024", "combined"], "function_name": "aime_2024", "is_alpha": false }, { "name": "AIME 2024 I", "description": "American Invitational Mathematics Examination 2024 (First)", "category": "math", "tags": ["math", "competition", "aime", "2024"], "function_name": "aime_2024_I", "is_alpha": false }, { "name": "AIME 2024 II", "description": "American Invitational Mathematics Examination 2024 (Second)", "category": "math", "tags": ["math", "competition", "aime", "2024"], "function_name": "aime_2024_II", "is_alpha": false }, { "name": "AIME 2025", "description": "American Invitational Mathematics Examination 2025", "category": "math", "tags": ["math", "competition", "aime", "2025"], "function_name": "aime_2025", "is_alpha": false }, { "name": "AIME 2025 II", "description": "American Invitational Mathematics Examination 2025 (Second)", "category": "math", "tags": ["math", "competition", "aime", "2025"], "function_name": "aime_2025_II", "is_alpha": false }, { "name": "ANLI (All Rounds)", "description": "Adversarial Natural Language Inference - challenging NLI benchmark", "category": "glue", "tags": ["multiple-choice", "nli", "adversarial", "reasoning"], "function_name": "anli", "is_alpha": false }, { "name": "ANLI Round 1", "description": "Adversarial NLI Round 1", "category": "glue", "tags": ["multiple-choice", "nli", "adversarial", "reasoning"], "function_name": "anli_r1", "is_alpha": false }, { "name": "ANLI Round 2", "description": "Adversarial NLI Round 2", "category": "glue", "tags": ["multiple-choice", "nli", "adversarial", "reasoning"], "function_name": "anli_r2", "is_alpha": false }, { "name": "ANLI Round 3", "description": "Adversarial NLI Round 3", "category": "glue", "tags": ["multiple-choice", "nli", "adversarial", "reasoning"], "function_name": "anli_r3", "is_alpha": false }, { "name": "ARC-AGI", "description": "Abstraction and Reasoning Corpus for Artificial General Intelligence; specify version with -T version=1 or version=2", "category": "core", "tags": ["reasoning", "pattern-recognition", "abstract-reasoning", "visual", "logic", "agi"], "function_name": "arc_agi", "is_alpha": false }, { "name": "ARC-AGI-1", "description": "ARC-AGI dataset version 1", "category": "core", "tags": ["reasoning", "pattern-recognition", "abstract-reasoning", "visual", "logic", "agi"], "function_name": "arc_agi_1", "is_alpha": false }, { "name": "ARC-AGI-2", "description": "ARC-AGI dataset version 2", "category": "core", "tags": ["reasoning", "pattern-recognition", "abstract-reasoning", "visual", "logic", "agi"], "function_name": "arc_agi_2", "is_alpha": false }, { "name": "ARC-Challenge", "description": "AI2 Reasoning Challenge - Challenging questions from grade-school science exams", "category": "core", "tags": ["multiple-choice", "science", "commonsense-reasoning"], "function_name": "arc_challenge", "is_alpha": false }, { "name": "ARC-Easy", "description": "AI2 Reasoning Challenge - Easy questions from grade-school science exams", "category": "core", "tags": ["multiple-choice", "science", "commonsense-reasoning"], "function_name": "arc_easy", "is_alpha": false }, { "name": "AgentDojo", "description": "AgentDojo - Evaluates AI robustness against prompt injection attacks from untrusted data and tools", "category": "ethics-social", "tags": ["safety", "prompt-injection", "adversarial", "agents"], "function_name": "agentdojo", "is_alpha": false }, { "name": "Arabic Exams (40 Subjects)", "description": "Multi-task Arabic language understanding benchmark from school exams across North Africa, the Levant, and the Gulf", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "multilingual", "education", "msa"], "function_name": "arabic_exams", "is_alpha": false }, { "name": "Arabic Exams: Accounting (University)", "description": "Arabic MMLU - Accounting questions from university-level exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "accounting", "university"], "function_name": "arabic_exams_accounting_university", "is_alpha": false }, { "name": "Arabic Exams: Arabic Language (General)", "description": "Arabic MMLU - Arabic language questions from general exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "language", "general"], "function_name": "arabic_exams_arabic_language_general", "is_alpha": false }, { "name": "Arabic Exams: Arabic Language (Grammar)", "description": "Arabic MMLU - Arabic language grammar questions", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "language", "grammar"], "function_name": "arabic_exams_arabic_language_grammar", "is_alpha": false }, { "name": "Arabic Exams: Arabic Language (High School)", "description": "Arabic MMLU - Arabic language questions from high school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "language", "high-school"], "function_name": "arabic_exams_arabic_language_high_school", "is_alpha": false }, { "name": "Arabic Exams: Arabic Language (Middle School)", "description": "Arabic MMLU - Arabic language questions from middle school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "language", "middle-school"], "function_name": "arabic_exams_arabic_language_middle_school", "is_alpha": false }, { "name": "Arabic Exams: Arabic Language (Primary School)", "description": "Arabic MMLU - Arabic language questions from primary school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "language", "primary-school"], "function_name": "arabic_exams_arabic_language_primary_school", "is_alpha": false }, { "name": "Arabic Exams: Biology (High School)", "description": "Arabic MMLU - Biology questions from high school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "biology", "high-school"], "function_name": "arabic_exams_biology_high_school", "is_alpha": false }, { "name": "Arabic Exams: Civics (High School)", "description": "Arabic MMLU - Civics questions from high school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "civics", "high-school"], "function_name": "arabic_exams_civics_high_school", "is_alpha": false }, { "name": "Arabic Exams: Civics (Middle School)", "description": "Arabic MMLU - Civics questions from middle school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "civics", "middle-school"], "function_name": "arabic_exams_civics_middle_school", "is_alpha": false }, { "name": "Arabic Exams: Computer Science (High School)", "description": "Arabic MMLU - Computer science questions from high school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "computer-science", "high-school"], "function_name": "arabic_exams_computer_science_high_school", "is_alpha": false }, { "name": "Arabic Exams: Computer Science (Middle School)", "description": "Arabic MMLU - Computer science questions from middle school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "computer-science", "middle-school"], "function_name": "arabic_exams_computer_science_middle_school", "is_alpha": false }, { "name": "Arabic Exams: Computer Science (Primary School)", "description": "Arabic MMLU - Computer science questions from primary school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "computer-science", "primary-school"], "function_name": "arabic_exams_computer_science_primary_school", "is_alpha": false }, { "name": "Arabic Exams: Computer Science (University)", "description": "Arabic MMLU - Computer science questions from university-level exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "computer-science", "university"], "function_name": "arabic_exams_computer_science_university", "is_alpha": false }, { "name": "Arabic Exams: Driving Test", "description": "Arabic MMLU - Driving test questions", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "driving"], "function_name": "arabic_exams_driving_test", "is_alpha": false }, { "name": "Arabic Exams: Economics (High School)", "description": "Arabic MMLU - Economics questions from high school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "economics", "high-school"], "function_name": "arabic_exams_economics_high_school", "is_alpha": false }, { "name": "Arabic Exams: Economics (Middle School)", "description": "Arabic MMLU - Economics questions from middle school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "economics", "middle-school"], "function_name": "arabic_exams_economics_middle_school", "is_alpha": false }, { "name": "Arabic Exams: Economics (University)", "description": "Arabic MMLU - Economics questions from university-level exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "economics", "university"], "function_name": "arabic_exams_economics_university", "is_alpha": false }, { "name": "Arabic Exams: General Knowledge", "description": "Arabic MMLU - General knowledge questions", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "general-knowledge"], "function_name": "arabic_exams_general_knowledge", "is_alpha": false }, { "name": "Arabic Exams: General Knowledge (Middle School)", "description": "Arabic MMLU - General knowledge questions from middle school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "general-knowledge", "middle-school"], "function_name": "arabic_exams_general_knowledge_middle_school", "is_alpha": false }, { "name": "Arabic Exams: General Knowledge (Primary School)", "description": "Arabic MMLU - General knowledge questions from primary school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "general-knowledge", "primary-school"], "function_name": "arabic_exams_general_knowledge_primary_school", "is_alpha": false }, { "name": "Arabic Exams: Geography (High School)", "description": "Arabic MMLU - Geography questions from high school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "geography", "high-school"], "function_name": "arabic_exams_geography_high_school", "is_alpha": false }, { "name": "Arabic Exams: Geography (Middle School)", "description": "Arabic MMLU - Geography questions from middle school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "geography", "middle-school"], "function_name": "arabic_exams_geography_middle_school", "is_alpha": false }, { "name": "Arabic Exams: Geography (Primary School)", "description": "Arabic MMLU - Geography questions from primary school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "geography", "primary-school"], "function_name": "arabic_exams_geography_primary_school", "is_alpha": false }, { "name": "Arabic Exams: History (High School)", "description": "Arabic MMLU - History questions from high school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "history", "high-school"], "function_name": "arabic_exams_history_high_school", "is_alpha": false }, { "name": "Arabic Exams: History (Middle School)", "description": "Arabic MMLU - History questions from middle school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "history", "middle-school"], "function_name": "arabic_exams_history_middle_school", "is_alpha": false }, { "name": "Arabic Exams: History (Primary School)", "description": "Arabic MMLU - History questions from primary school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "history", "primary-school"], "function_name": "arabic_exams_history_primary_school", "is_alpha": false }, { "name": "Arabic Exams: Islamic Studies (General)", "description": "Arabic MMLU - Islamic studies questions from general exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "islamic-studies", "general"], "function_name": "arabic_exams_islamic_studies_general", "is_alpha": false }, { "name": "Arabic Exams: Islamic Studies (High School)", "description": "Arabic MMLU - Islamic studies questions from high school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "islamic-studies", "high-school"], "function_name": "arabic_exams_islamic_studies_high_school", "is_alpha": false }, { "name": "Arabic Exams: Islamic Studies (Middle School)", "description": "Arabic MMLU - Islamic studies questions from middle school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "islamic-studies", "middle-school"], "function_name": "arabic_exams_islamic_studies_middle_school", "is_alpha": false }, { "name": "Arabic Exams: Islamic Studies (Primary School)", "description": "Arabic MMLU - Islamic studies questions from primary school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "islamic-studies", "primary-school"], "function_name": "arabic_exams_islamic_studies_primary_school", "is_alpha": false }, { "name": "Arabic Exams: Law (Professional)", "description": "Arabic MMLU - Law questions from professional exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "law", "professional"], "function_name": "arabic_exams_law_professional", "is_alpha": false }, { "name": "Arabic Exams: Management (University)", "description": "Arabic MMLU - Management questions from university-level exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "management", "university"], "function_name": "arabic_exams_management_university", "is_alpha": false }, { "name": "Arabic Exams: Math (High School)", "description": "Arabic MMLU - Math questions from high school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "math", "high-school"], "function_name": "arabic_exams_math_high_school", "is_alpha": false }, { "name": "Arabic Exams: Math (Primary School)", "description": "Arabic MMLU - Math questions from primary school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "math", "primary-school"], "function_name": "arabic_exams_math_primary_school", "is_alpha": false }, { "name": "Arabic Exams: Natural Science (Middle School)", "description": "Arabic MMLU - Natural science questions from middle school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "natural-science", "middle-school"], "function_name": "arabic_exams_natural_science_middle_school", "is_alpha": false }, { "name": "Arabic Exams: Natural Science (Primary School)", "description": "Arabic MMLU - Natural science questions from primary school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "natural-science", "primary-school"], "function_name": "arabic_exams_natural_science_primary_school", "is_alpha": false }, { "name": "Arabic Exams: Philosophy (High School)", "description": "Arabic MMLU - Philosophy questions from high school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "philosophy", "high-school"], "function_name": "arabic_exams_philosophy_high_school", "is_alpha": false }, { "name": "Arabic Exams: Physics (High School)", "description": "Arabic MMLU - Physics questions from high school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "physics", "high-school"], "function_name": "arabic_exams_physics_high_school", "is_alpha": false }, { "name": "Arabic Exams: Political Science (University)", "description": "Arabic MMLU - Political science questions from university-level exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "political-science", "university"], "function_name": "arabic_exams_political_science_university", "is_alpha": false }, { "name": "Arabic Exams: Social Science (Middle School)", "description": "Arabic MMLU - Social science questions from middle school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "social-science", "middle-school"], "function_name": "arabic_exams_social_science_middle_school", "is_alpha": false }, { "name": "Arabic Exams: Social Science (Primary School)", "description": "Arabic MMLU - Social science questions from primary school exams", "category": "domain-specific", "tags": ["multiple-choice", "arabic", "social-science", "primary-school"], "function_name": "arabic_exams_social_science_primary_school", "is_alpha": false }, { "name": "BBH: Causal Judgment", "description": "BigBench Hard - Causal judgment reasoning", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought"], "function_name": "bbh_causal_judgment", "is_alpha": false }, { "name": "BBH: Date Understanding", "description": "BigBench Hard - Understanding and reasoning about dates", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought"], "function_name": "bbh_date_understanding", "is_alpha": false }, { "name": "BBH: Disambiguation QA", "description": "BigBench Hard - Pronoun disambiguation in questions", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought"], "function_name": "bbh_disambiguation_qa", "is_alpha": false }, { "name": "BBH: Geometric Shapes", "description": "BigBench Hard - Reasoning about geometric shapes", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "geometry"], "function_name": "bbh_geometric_shapes", "is_alpha": false }, { "name": "BBH: Logical Deduction (3 Objects)", "description": "BigBench Hard - Logical deduction with three objects", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "logic"], "function_name": "bbh_logical_deduction_three_objects", "is_alpha": false }, { "name": "BBH: Logical Deduction (5 Objects)", "description": "BigBench Hard - Logical deduction with five objects", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "logic"], "function_name": "bbh_logical_deduction_five_objects", "is_alpha": false }, { "name": "BBH: Logical Deduction (7 Objects)", "description": "BigBench Hard - Logical deduction with seven objects", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "logic"], "function_name": "bbh_logical_deduction_seven_objects", "is_alpha": false }, { "name": "BBH: Movie Recommendation", "description": "BigBench Hard - Movie recommendation reasoning", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought"], "function_name": "bbh_movie_recommendation", "is_alpha": false }, { "name": "BBH: Navigate", "description": "BigBench Hard - Spatial navigation reasoning", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "spatial"], "function_name": "bbh_navigate", "is_alpha": false }, { "name": "BBH: Reasoning About Colored Objects", "description": "BigBench Hard - Reasoning about colored objects", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought"], "function_name": "bbh_reasoning_about_colored_objects", "is_alpha": false }, { "name": "BBH: Ruin Names", "description": "BigBench Hard - Word manipulation and reasoning", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "wordplay"], "function_name": "bbh_ruin_names", "is_alpha": false }, { "name": "BBH: Salient Translation Error Detection", "description": "BigBench Hard - Detecting translation errors", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "translation"], "function_name": "bbh_salient_translation_error_detection", "is_alpha": false }, { "name": "BBH: Snarks", "description": "BigBench Hard - Understanding sarcasm and irony", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "sarcasm"], "function_name": "bbh_snarks", "is_alpha": false }, { "name": "BBH: Sports Understanding", "description": "BigBench Hard - Sports knowledge and reasoning", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "sports"], "function_name": "bbh_sports_understanding", "is_alpha": false }, { "name": "BBH: Temporal Sequences", "description": "BigBench Hard - Understanding temporal sequences", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "temporal"], "function_name": "bbh_temporal_sequences", "is_alpha": false }, { "name": "BBH: Tracking Shuffled Objects (3 Objects)", "description": "BigBench Hard - Tracking three shuffled objects", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "tracking"], "function_name": "bbh_tracking_shuffled_objects_three_objects", "is_alpha": false }, { "name": "BBH: Tracking Shuffled Objects (5 Objects)", "description": "BigBench Hard - Tracking five shuffled objects", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "tracking"], "function_name": "bbh_tracking_shuffled_objects_five_objects", "is_alpha": false }, { "name": "BBH: Tracking Shuffled Objects (7 Objects)", "description": "BigBench Hard - Tracking seven shuffled objects", "category": "core", "tags": ["multiple-choice", "reasoning", "bigbench", "chain-of-thought", "tracking"], "function_name": "bbh_tracking_shuffled_objects_seven_objects", "is_alpha": false }, { "name": "BBQ (Main Function)", "description": "BBQ bias evaluation for a specific category - use individual category tasks instead", "category": "ethics-social", "tags": ["multiple-choice", "bias", "fairness", "social-bias", "qa"], "function_name": "bbq", "is_alpha": false }, { "name": "BBQ: Age", "description": "Evaluate age-related biases in question-answering", "category": "ethics-social", "tags": ["multiple-choice", "bias", "fairness", "social-bias", "qa", "age"], "function_name": "bbq_age", "is_alpha": false }, { "name": "BBQ: Disability Status", "description": "Evaluate disability-related biases in question-answering", "category": "ethics-social", "tags": ["multiple-choice", "bias", "fairness", "social-bias", "qa", "disability"], "function_name": "bbq_disability_status", "is_alpha": false }, { "name": "BBQ: Gender Identity", "description": "Evaluate gender identity-related biases in question-answering", "category": "ethics-social", "tags": ["multiple-choice", "bias", "fairness", "social-bias", "qa", "gender"], "function_name": "bbq_gender_identity", "is_alpha": false }, { "name": "BBQ: Nationality", "description": "Evaluate nationality-related biases in question-answering", "category": "ethics-social", "tags": ["multiple-choice", "bias", "fairness", "social-bias", "qa", "nationality"], "function_name": "bbq_nationality", "is_alpha": false }, { "name": "BBQ: Physical Appearance", "description": "Evaluate physical appearance-related biases in question-answering", "category": "ethics-social", "tags": ["multiple-choice", "bias", "fairness", "social-bias", "qa", "appearance"], "function_name": "bbq_physical_appearance", "is_alpha": false }, { "name": "BBQ: Race × Gender", "description": "Evaluate intersectional race and gender biases", "category": "ethics-social", "tags": ["multiple-choice", "bias", "fairness", "social-bias", "qa", "race", "gender"], "function_name": "bbq_race_x_gender", "is_alpha": false }, { "name": "BBQ: Race × SES", "description": "Evaluate intersectional race and socioeconomic status biases", "category": "ethics-social", "tags": ["multiple-choice", "bias", "fairness", "social-bias", "qa", "race", "ses"], "function_name": "bbq_race_x_ses", "is_alpha": false }, { "name": "BBQ: Race/Ethnicity", "description": "Evaluate race and ethnicity-related biases in question-answering", "category": "ethics-social", "tags": ["multiple-choice", "bias", "fairness", "social-bias", "qa", "race"], "function_name": "bbq_race_ethnicity", "is_alpha": false }, { "name": "BBQ: Religion", "description": "Evaluate religion-related biases in question-answering", "category": "ethics-social", "tags": ["multiple-choice", "bias", "fairness", "social-bias", "qa", "religion"], "function_name": "bbq_religion", "is_alpha": false }, { "name": "BBQ: Sexual Orientation", "description": "Evaluate sexual orientation-related biases in question-answering", "category": "ethics-social", "tags": ["multiple-choice", "bias", "fairness", "social-bias", "qa", "sexual-orientation"], "function_name": "bbq_sexual_orientation", "is_alpha": false }, { "name": "BBQ: Socioeconomic Status", "description": "Evaluate socioeconomic status-related biases in question-answering", "category": "ethics-social", "tags": ["multiple-choice", "bias", "fairness", "social-bias", "qa", "ses"], "function_name": "bbq_ses", "is_alpha": false }, { "name": "BLiMP (67 Linguistic Phenomena)", "description": "Benchmark of Linguistic Minimal Pairs testing grammatical knowledge through minimal pair comparisons", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "morphology"], "function_name": "blimp", "is_alpha": false }, { "name": "BLiMP: 'Only' NPI scope", "description": "BLiMP 'Only' NPI scope", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_only_npi_scope", "is_alpha": false }, { "name": "BLiMP: 'Only' as NPI licensor", "description": "BLiMP 'Only' as NPI licensor", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_only_npi_licensor_present", "is_alpha": false }, { "name": "BLiMP: Adjunct island effects", "description": "BLiMP Adjunct island effects", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_adjunct_island", "is_alpha": false }, { "name": "BLiMP: Anaphor gender agreement", "description": "BLiMP Anaphor gender agreement", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_anaphor_gender_agreement", "is_alpha": false }, { "name": "BLiMP: Anaphor number agreement", "description": "BLiMP Anaphor number agreement", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_anaphor_number_agreement", "is_alpha": false }, { "name": "BLiMP: Animate subject in passive constructions", "description": "BLiMP Animate subject in passive constructions", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_animate_subject_passive", "is_alpha": false }, { "name": "BLiMP: Animate subject in transitive constructions", "description": "BLiMP Animate subject in transitive constructions", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_animate_subject_trans", "is_alpha": false }, { "name": "BLiMP: Binding Principle A - c-command", "description": "BLiMP Binding Principle A - c-command", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_principle_A_c_command", "is_alpha": false }, { "name": "BLiMP: Binding Principle A - case (1)", "description": "BLiMP Binding Principle A - case (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_principle_A_case_1", "is_alpha": false }, { "name": "BLiMP: Binding Principle A - case (2)", "description": "BLiMP Binding Principle A - case (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_principle_A_case_2", "is_alpha": false }, { "name": "BLiMP: Binding Principle A - domain (1)", "description": "BLiMP Binding Principle A - domain (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_principle_A_domain_1", "is_alpha": false }, { "name": "BLiMP: Binding Principle A - domain (2)", "description": "BLiMP Binding Principle A - domain (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_principle_A_domain_2", "is_alpha": false }, { "name": "BLiMP: Binding Principle A - domain (3)", "description": "BLiMP Binding Principle A - domain (3)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_principle_A_domain_3", "is_alpha": false }, { "name": "BLiMP: Binding Principle A - reconstruction", "description": "BLiMP Binding Principle A - reconstruction", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_principle_A_reconstruction", "is_alpha": false }, { "name": "BLiMP: Causative constructions", "description": "BLiMP Causative constructions", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_causative", "is_alpha": false }, { "name": "BLiMP: Complex NP island effects", "description": "BLiMP Complex NP island effects", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_complex_NP_island", "is_alpha": false }, { "name": "BLiMP: Coordinate structure constraint - complex left branch", "description": "BLiMP Coordinate structure constraint - complex left branch", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_coordinate_structure_constraint_complex_left_branch", "is_alpha": false }, { "name": "BLiMP: Coordinate structure constraint - object extraction", "description": "BLiMP Coordinate structure constraint - object extraction", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_coordinate_structure_constraint_object_extraction", "is_alpha": false }, { "name": "BLiMP: Determiner-noun agreement (1)", "description": "BLiMP Determiner-noun agreement (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_determiner_noun_agreement_1", "is_alpha": false }, { "name": "BLiMP: Determiner-noun agreement (2)", "description": "BLiMP Determiner-noun agreement (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_determiner_noun_agreement_2", "is_alpha": false }, { "name": "BLiMP: Determiner-noun agreement with adjective (1)", "description": "BLiMP Determiner-noun agreement with adjective (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_determiner_noun_agreement_with_adjective_1", "is_alpha": false }, { "name": "BLiMP: Determiner-noun agreement with adjective (2)", "description": "BLiMP Determiner-noun agreement with adjective (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_determiner_noun_agreement_with_adj_2", "is_alpha": false }, { "name": "BLiMP: Determiner-noun agreement with adjective and irregular nouns (1)", "description": "BLiMP Determiner-noun agreement with adjective and irregular nouns (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_determiner_noun_agreement_with_adj_irregular_1", "is_alpha": false }, { "name": "BLiMP: Determiner-noun agreement with adjective and irregular nouns (2)", "description": "BLiMP Determiner-noun agreement with adjective and irregular nouns (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_determiner_noun_agreement_with_adj_irregular_2", "is_alpha": false }, { "name": "BLiMP: Determiner-noun agreement with irregular nouns (1)", "description": "BLiMP Determiner-noun agreement with irregular nouns (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_determiner_noun_agreement_irregular_1", "is_alpha": false }, { "name": "BLiMP: Determiner-noun agreement with irregular nouns (2)", "description": "BLiMP Determiner-noun agreement with irregular nouns (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_determiner_noun_agreement_irregular_2", "is_alpha": false }, { "name": "BLiMP: Distractor agreement in relative clauses", "description": "BLiMP Distractor agreement in relative clauses", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_distractor_agreement_relative_clause", "is_alpha": false }, { "name": "BLiMP: Distractor agreement with relational nouns", "description": "BLiMP Distractor agreement with relational nouns", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_distractor_agreement_relational_noun", "is_alpha": false }, { "name": "BLiMP: Dropped argument", "description": "BLiMP Dropped argument", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_drop_argument", "is_alpha": false }, { "name": "BLiMP: Existential 'there' with object raising", "description": "BLiMP Existential 'there' with object raising", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_existential_there_object_raising", "is_alpha": false }, { "name": "BLiMP: Existential 'there' with quantifiers (1)", "description": "BLiMP Existential 'there' with quantifiers (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_existential_there_quantifiers_1", "is_alpha": false }, { "name": "BLiMP: Existential 'there' with quantifiers (2)", "description": "BLiMP Existential 'there' with quantifiers (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_existential_there_quantifiers_2", "is_alpha": false }, { "name": "BLiMP: Existential 'there' with subject raising", "description": "BLiMP Existential 'there' with subject raising", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_existential_there_subject_raising", "is_alpha": false }, { "name": "BLiMP: Expletive 'it' with object raising", "description": "BLiMP Expletive 'it' with object raising", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_expletive_it_object_raising", "is_alpha": false }, { "name": "BLiMP: Inchoative constructions", "description": "BLiMP Inchoative constructions", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_inchoative", "is_alpha": false }, { "name": "BLiMP: Intransitive verbs", "description": "BLiMP Intransitive verbs", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_intransitive", "is_alpha": false }, { "name": "BLiMP: Irregular past participles as adjectives", "description": "BLiMP Irregular past participles as adjectives", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_irregular_past_participle_adjectives", "is_alpha": false }, { "name": "BLiMP: Irregular past participles in verbs", "description": "BLiMP Irregular past participles in verbs", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_irregular_past_participle_verbs", "is_alpha": false }, { "name": "BLiMP: Left branch island effects in echo questions", "description": "BLiMP Left branch island effects in echo questions", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_left_branch_island_echo_question", "is_alpha": false }, { "name": "BLiMP: Left branch island effects in simple questions", "description": "BLiMP Left branch island effects in simple questions", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_left_branch_island_simple_question", "is_alpha": false }, { "name": "BLiMP: Matrix question NPI licensor present", "description": "BLiMP Matrix question NPI licensor present", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_matrix_question_npi_licensor_present", "is_alpha": false }, { "name": "BLiMP: N-bar ellipsis (1)", "description": "BLiMP N-bar ellipsis (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_ellipsis_n_bar_1", "is_alpha": false }, { "name": "BLiMP: N-bar ellipsis (2)", "description": "BLiMP N-bar ellipsis (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_ellipsis_n_bar_2", "is_alpha": false }, { "name": "BLiMP: Negative polarity items present (1)", "description": "BLiMP Negative polarity items present (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_npi_present_1", "is_alpha": false }, { "name": "BLiMP: Negative polarity items present (2)", "description": "BLiMP Negative polarity items present (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_npi_present_2", "is_alpha": false }, { "name": "BLiMP: Passive constructions (1)", "description": "BLiMP Passive constructions (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_passive_1", "is_alpha": false }, { "name": "BLiMP: Passive constructions (2)", "description": "BLiMP Passive constructions (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_passive_2", "is_alpha": false }, { "name": "BLiMP: Sentential negation NPI scope", "description": "BLiMP Sentential negation NPI scope", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_sentential_negation_npi_scope", "is_alpha": false }, { "name": "BLiMP: Sentential negation as NPI licensor", "description": "BLiMP Sentential negation as NPI licensor", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_sentential_negation_npi_licensor_present", "is_alpha": false }, { "name": "BLiMP: Sentential subject island effects", "description": "BLiMP Sentential subject island effects", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_sentential_subject_island", "is_alpha": false }, { "name": "BLiMP: Subject-verb agreement with irregular plurals (1)", "description": "BLiMP Subject-verb agreement with irregular plurals (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_irregular_plural_subject_verb_agreement_1", "is_alpha": false }, { "name": "BLiMP: Subject-verb agreement with irregular plurals (2)", "description": "BLiMP Subject-verb agreement with irregular plurals (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_irregular_plural_subject_verb_agreement_2", "is_alpha": false }, { "name": "BLiMP: Subject-verb agreement with regular plurals (1)", "description": "BLiMP Subject-verb agreement with regular plurals (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_regular_plural_subject_verb_agreement_1", "is_alpha": false }, { "name": "BLiMP: Subject-verb agreement with regular plurals (2)", "description": "BLiMP Subject-verb agreement with regular plurals (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_regular_plural_subject_verb_agreement_2", "is_alpha": false }, { "name": "BLiMP: Superlative quantifiers (1)", "description": "BLiMP Superlative quantifiers (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_superlative_quantifiers_1", "is_alpha": false }, { "name": "BLiMP: Superlative quantifiers (2)", "description": "BLiMP Superlative quantifiers (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_superlative_quantifiers_2", "is_alpha": false }, { "name": "BLiMP: Tough vs raising constructions (1)", "description": "BLiMP Tough vs raising constructions (1)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_tough_vs_raising_1", "is_alpha": false }, { "name": "BLiMP: Tough vs raising constructions (2)", "description": "BLiMP Tough vs raising constructions (2)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_tough_vs_raising_2", "is_alpha": false }, { "name": "BLiMP: Transitive verbs", "description": "BLiMP Transitive verbs", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_transitive", "is_alpha": false }, { "name": "BLiMP: Wh vs that complementizers with gap", "description": "BLiMP Wh vs that complementizers with gap", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_wh_vs_that_with_gap", "is_alpha": false }, { "name": "BLiMP: Wh vs that complementizers with gap (long-distance)", "description": "BLiMP Wh vs that complementizers with gap (long-distance)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_wh_vs_that_with_gap_long_distance", "is_alpha": false }, { "name": "BLiMP: Wh vs that complementizers without gap", "description": "BLiMP Wh vs that complementizers without gap", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_wh_vs_that_no_gap", "is_alpha": false }, { "name": "BLiMP: Wh vs that complementizers without gap (long-distance)", "description": "BLiMP Wh vs that complementizers without gap (long-distance)", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_wh_vs_that_no_gap_long_distance", "is_alpha": false }, { "name": "BLiMP: Wh-island effects", "description": "BLiMP Wh-island effects", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_wh_island", "is_alpha": false }, { "name": "BLiMP: Wh-questions with long-distance subject gap", "description": "BLiMP Wh-questions with long-distance subject gap", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_wh_questions_subject_gap_long_distance", "is_alpha": false }, { "name": "BLiMP: Wh-questions with object gap", "description": "BLiMP Wh-questions with object gap", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_wh_questions_object_gap", "is_alpha": false }, { "name": "BLiMP: Wh-questions with subject gap", "description": "BLiMP Wh-questions with subject gap", "category": "linguistic", "tags": ["multiple-choice", "linguistics", "grammar", "syntax", "blimp"], "function_name": "blimp_wh_questions_subject_gap", "is_alpha": false }, { "name": "BRUMO 2025", "description": "Bruno Mathematical Olympiad 2025", "category": "math", "tags": ["math", "competition", "olympiad", "2025"], "function_name": "brumo_2025", "is_alpha": false }, { "name": "BigBench: Anachronisms", "description": "BigBench MCQ task: anachronisms", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_anachronisms", "is_alpha": false }, { "name": "BigBench: Analogical Similarity", "description": "BigBench MCQ task: analogical_similarity", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_analogical_similarity", "is_alpha": false }, { "name": "BigBench: Analytic Entailment", "description": "BigBench MCQ task: analytic_entailment", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_analytic_entailment", "is_alpha": false }, { "name": "BigBench: Arithmetic", "description": "BigBench MCQ task: arithmetic", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_arithmetic", "is_alpha": false }, { "name": "BigBench: Authorship Verification", "description": "BigBench MCQ task: authorship_verification", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_authorship_verification", "is_alpha": false }, { "name": "BigBench: Bbq Lite Json", "description": "BigBench MCQ task: bbq_lite_json", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_bbq_lite_json", "is_alpha": false }, { "name": "BigBench: Causal Judgment", "description": "BigBench MCQ task: causal_judgment", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_causal_judgment", "is_alpha": false }, { "name": "BigBench: Cause And Effect", "description": "BigBench MCQ task: cause_and_effect", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_cause_and_effect", "is_alpha": false }, { "name": "BigBench: Checkmate In One", "description": "BigBench MCQ task: checkmate_in_one", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_checkmate_in_one", "is_alpha": false }, { "name": "BigBench: Cifar10 Classification", "description": "BigBench MCQ task: cifar10_classification", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_cifar10_classification", "is_alpha": false }, { "name": "BigBench: Code Line Description", "description": "BigBench MCQ task: code_line_description", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_code_line_description", "is_alpha": false }, { "name": "BigBench: Color", "description": "BigBench MCQ task: color", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_color", "is_alpha": false }, { "name": "BigBench: Common Morpheme", "description": "BigBench MCQ task: common_morpheme", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_common_morpheme", "is_alpha": false }, { "name": "BigBench: Conceptual Combinations", "description": "BigBench MCQ task: conceptual_combinations", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_conceptual_combinations", "is_alpha": false }, { "name": "BigBench: Contextual Parametric Knowledge Conflicts", "description": "BigBench MCQ task: contextual_parametric_knowledge_conflicts", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_contextual_parametric_knowledge_conflicts", "is_alpha": false }, { "name": "BigBench: Crash Blossom", "description": "BigBench MCQ task: crash_blossom", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_crash_blossom", "is_alpha": false }, { "name": "BigBench: Crass Ai", "description": "BigBench MCQ task: crass_ai", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_crass_ai", "is_alpha": false }, { "name": "BigBench: Cryobiology Spanish", "description": "BigBench MCQ task: cryobiology_spanish", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_cryobiology_spanish", "is_alpha": false }, { "name": "BigBench: Cs Algorithms", "description": "BigBench MCQ task: cs_algorithms", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_cs_algorithms", "is_alpha": false }, { "name": "BigBench: Dark Humor Detection", "description": "BigBench MCQ task: dark_humor_detection", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_dark_humor_detection", "is_alpha": false }, { "name": "BigBench: Date Understanding", "description": "BigBench MCQ task: date_understanding", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_date_understanding", "is_alpha": false }, { "name": "BigBench: Disambiguation Qa", "description": "BigBench MCQ task: disambiguation_qa", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_disambiguation_qa", "is_alpha": false }, { "name": "BigBench: Discourse Marker Prediction", "description": "BigBench MCQ task: discourse_marker_prediction", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_discourse_marker_prediction", "is_alpha": false }, { "name": "BigBench: Dyck Languages", "description": "BigBench MCQ task: dyck_languages", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_dyck_languages", "is_alpha": false }, { "name": "BigBench: Elementary Math Qa", "description": "BigBench MCQ task: elementary_math_qa", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_elementary_math_qa", "is_alpha": false }, { "name": "BigBench: Emoji Movie", "description": "BigBench MCQ task: emoji_movie", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_emoji_movie", "is_alpha": false }, { "name": "BigBench: Emojis Emotion Prediction", "description": "BigBench MCQ task: emojis_emotion_prediction", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_emojis_emotion_prediction", "is_alpha": false }, { "name": "BigBench: Empirical Judgments", "description": "BigBench MCQ task: empirical_judgments", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_empirical_judgments", "is_alpha": false }, { "name": "BigBench: English Proverbs", "description": "BigBench MCQ task: english_proverbs", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_english_proverbs", "is_alpha": false }, { "name": "BigBench: English Russian Proverbs", "description": "BigBench MCQ task: english_russian_proverbs", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_english_russian_proverbs", "is_alpha": false }, { "name": "BigBench: Entailed Polarity", "description": "BigBench MCQ task: entailed_polarity", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_entailed_polarity", "is_alpha": false }, { "name": "BigBench: Entailed Polarity Hindi", "description": "BigBench MCQ task: entailed_polarity_hindi", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_entailed_polarity_hindi", "is_alpha": false }, { "name": "BigBench: Epistemic Reasoning", "description": "BigBench MCQ task: epistemic_reasoning", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_epistemic_reasoning", "is_alpha": false }, { "name": "BigBench: Evaluating Information Essentiality", "description": "BigBench MCQ task: evaluating_information_essentiality", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_evaluating_information_essentiality", "is_alpha": false }, { "name": "BigBench: Fact Checker", "description": "BigBench MCQ task: fact_checker", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_fact_checker", "is_alpha": false }, { "name": "BigBench: Fantasy Reasoning", "description": "BigBench MCQ task: fantasy_reasoning", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_fantasy_reasoning", "is_alpha": false }, { "name": "BigBench: Figure Of Speech Detection", "description": "BigBench MCQ task: figure_of_speech_detection", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_figure_of_speech_detection", "is_alpha": false }, { "name": "BigBench: Formal Fallacies Syllogisms Negation", "description": "BigBench MCQ task: formal_fallacies_syllogisms_negation", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_formal_fallacies_syllogisms_negation", "is_alpha": false }, { "name": "BigBench: General Knowledge", "description": "BigBench MCQ task: general_knowledge", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_general_knowledge", "is_alpha": false }, { "name": "BigBench: Geometric Shapes", "description": "BigBench MCQ task: geometric_shapes", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_geometric_shapes", "is_alpha": false }, { "name": "BigBench: Goal Step Wikihow", "description": "BigBench MCQ task: goal_step_wikihow", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_goal_step_wikihow", "is_alpha": false }, { "name": "BigBench: Gre Reading Comprehension", "description": "BigBench MCQ task: gre_reading_comprehension", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_gre_reading_comprehension", "is_alpha": false }, { "name": "BigBench: Hhh Alignment", "description": "BigBench MCQ task: hhh_alignment", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_hhh_alignment", "is_alpha": false }, { "name": "BigBench: Hindu Knowledge", "description": "BigBench MCQ task: hindu_knowledge", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_hindu_knowledge", "is_alpha": false }, { "name": "BigBench: Hinglish Toxicity", "description": "BigBench MCQ task: hinglish_toxicity", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_hinglish_toxicity", "is_alpha": false }, { "name": "BigBench: Human Organs Senses", "description": "BigBench MCQ task: human_organs_senses", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_human_organs_senses", "is_alpha": false }, { "name": "BigBench: Hyperbaton", "description": "BigBench MCQ task: hyperbaton", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_hyperbaton", "is_alpha": false }, { "name": "BigBench: Identify Math Theorems", "description": "BigBench MCQ task: identify_math_theorems", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_identify_math_theorems", "is_alpha": false }, { "name": "BigBench: Identify Odd Metaphor", "description": "BigBench MCQ task: identify_odd_metaphor", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_identify_odd_metaphor", "is_alpha": false }, { "name": "BigBench: Implicatures", "description": "BigBench MCQ task: implicatures", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_implicatures", "is_alpha": false }, { "name": "BigBench: Implicit Relations", "description": "BigBench MCQ task: implicit_relations", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_implicit_relations", "is_alpha": false }, { "name": "BigBench: Indic Cause And Effect", "description": "BigBench MCQ task: indic_cause_and_effect", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_indic_cause_and_effect", "is_alpha": false }, { "name": "BigBench: Intent Recognition", "description": "BigBench MCQ task: intent_recognition", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_intent_recognition", "is_alpha": false }, { "name": "BigBench: International Phonetic Alphabet Nli", "description": "BigBench MCQ task: international_phonetic_alphabet_nli", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_international_phonetic_alphabet_nli", "is_alpha": false }, { "name": "BigBench: Intersect Geometry", "description": "BigBench MCQ task: intersect_geometry", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_intersect_geometry", "is_alpha": false }, { "name": "BigBench: Irony Identification", "description": "BigBench MCQ task: irony_identification", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_irony_identification", "is_alpha": false }, { "name": "BigBench: Kanji Ascii", "description": "BigBench MCQ task: kanji_ascii", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_kanji_ascii", "is_alpha": false }, { "name": "BigBench: Kannada", "description": "BigBench MCQ task: kannada", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_kannada", "is_alpha": false }, { "name": "BigBench: Key Value Maps", "description": "BigBench MCQ task: key_value_maps", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_key_value_maps", "is_alpha": false }, { "name": "BigBench: Known Unknowns", "description": "BigBench MCQ task: known_unknowns", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_known_unknowns", "is_alpha": false }, { "name": "BigBench: Language Identification", "description": "BigBench MCQ task: language_identification", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_language_identification", "is_alpha": false }, { "name": "BigBench: Logic Grid Puzzle", "description": "BigBench MCQ task: logic_grid_puzzle", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_logic_grid_puzzle", "is_alpha": false }, { "name": "BigBench: Logical Args", "description": "BigBench MCQ task: logical_args", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_logical_args", "is_alpha": false }, { "name": "BigBench: Logical Deduction", "description": "BigBench MCQ task: logical_deduction", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_logical_deduction", "is_alpha": false }, { "name": "BigBench: Logical Fallacy Detection", "description": "BigBench MCQ task: logical_fallacy_detection", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_logical_fallacy_detection", "is_alpha": false }, { "name": "BigBench: Logical Sequence", "description": "BigBench MCQ task: logical_sequence", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_logical_sequence", "is_alpha": false }, { "name": "BigBench: Mathematical Induction", "description": "BigBench MCQ task: mathematical_induction", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_mathematical_induction", "is_alpha": false }, { "name": "BigBench: Medical Questions Russian", "description": "BigBench MCQ task: medical_questions_russian", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_medical_questions_russian", "is_alpha": false }, { "name": "BigBench: Metaphor Boolean", "description": "BigBench MCQ task: metaphor_boolean", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_metaphor_boolean", "is_alpha": false }, { "name": "BigBench: Metaphor Understanding", "description": "BigBench MCQ task: metaphor_understanding", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_metaphor_understanding", "is_alpha": false }, { "name": "BigBench: Minute Mysteries Qa", "description": "BigBench MCQ task: minute_mysteries_qa", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_minute_mysteries_qa", "is_alpha": false }, { "name": "BigBench: Misconceptions", "description": "BigBench MCQ task: misconceptions", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_misconceptions", "is_alpha": false }, { "name": "BigBench: Misconceptions Russian", "description": "BigBench MCQ task: misconceptions_russian", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_misconceptions_russian", "is_alpha": false }, { "name": "BigBench: Mnist Ascii", "description": "BigBench MCQ task: mnist_ascii", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_mnist_ascii", "is_alpha": false }, { "name": "BigBench: Moral Permissibility", "description": "BigBench MCQ task: moral_permissibility", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_moral_permissibility", "is_alpha": false }, { "name": "BigBench: Movie Dialog Same Or Different", "description": "BigBench MCQ task: movie_dialog_same_or_different", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_movie_dialog_same_or_different", "is_alpha": false }, { "name": "BigBench: Movie Recommendation", "description": "BigBench MCQ task: movie_recommendation", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_movie_recommendation", "is_alpha": false }, { "name": "BigBench: Navigate", "description": "BigBench MCQ task: navigate", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_navigate", "is_alpha": false }, { "name": "BigBench: Nonsense Words Grammar", "description": "BigBench MCQ task: nonsense_words_grammar", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_nonsense_words_grammar", "is_alpha": false }, { "name": "BigBench: Novel Concepts", "description": "BigBench MCQ task: novel_concepts", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_novel_concepts", "is_alpha": false }, { "name": "BigBench: Odd One Out", "description": "BigBench MCQ task: odd_one_out", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_odd_one_out", "is_alpha": false }, { "name": "BigBench: Parsinlu Qa", "description": "BigBench MCQ task: parsinlu_qa", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_parsinlu_qa", "is_alpha": false }, { "name": "BigBench: Penguins In A Table", "description": "BigBench MCQ task: penguins_in_a_table", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_penguins_in_a_table", "is_alpha": false }, { "name": "BigBench: Periodic Elements", "description": "BigBench MCQ task: periodic_elements", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_periodic_elements", "is_alpha": false }, { "name": "BigBench: Persian Idioms", "description": "BigBench MCQ task: persian_idioms", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_persian_idioms", "is_alpha": false }, { "name": "BigBench: Phrase Relatedness", "description": "BigBench MCQ task: phrase_relatedness", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_phrase_relatedness", "is_alpha": false }, { "name": "BigBench: Physical Intuition", "description": "BigBench MCQ task: physical_intuition", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_physical_intuition", "is_alpha": false }, { "name": "BigBench: Physics", "description": "BigBench MCQ task: physics", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_physics", "is_alpha": false }, { "name": "BigBench: Play Dialog Same Or Different", "description": "BigBench MCQ task: play_dialog_same_or_different", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_play_dialog_same_or_different", "is_alpha": false }, { "name": "BigBench: Presuppositions As Nli", "description": "BigBench MCQ task: presuppositions_as_nli", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_presuppositions_as_nli", "is_alpha": false }, { "name": "BigBench: Question Selection", "description": "BigBench MCQ task: question_selection", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_question_selection", "is_alpha": false }, { "name": "BigBench: Real Or Fake Text", "description": "BigBench MCQ task: real_or_fake_text", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_real_or_fake_text", "is_alpha": false }, { "name": "BigBench: Reasoning About Colored Objects", "description": "BigBench MCQ task: reasoning_about_colored_objects", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_reasoning_about_colored_objects", "is_alpha": false }, { "name": "BigBench: Rhyming", "description": "BigBench MCQ task: rhyming", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_rhyming", "is_alpha": false }, { "name": "BigBench: Riddle Sense", "description": "BigBench MCQ task: riddle_sense", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_riddle_sense", "is_alpha": false }, { "name": "BigBench: Ruin Names", "description": "BigBench MCQ task: ruin_names", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_ruin_names", "is_alpha": false }, { "name": "BigBench: Salient Translation Error Detection", "description": "BigBench MCQ task: salient_translation_error_detection", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_salient_translation_error_detection", "is_alpha": false }, { "name": "BigBench: Sentence Ambiguity", "description": "BigBench MCQ task: sentence_ambiguity", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_sentence_ambiguity", "is_alpha": false }, { "name": "BigBench: Similarities Abstraction", "description": "BigBench MCQ task: similarities_abstraction", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_similarities_abstraction", "is_alpha": false }, { "name": "BigBench: Simple Ethical Questions", "description": "BigBench MCQ task: simple_ethical_questions", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_simple_ethical_questions", "is_alpha": false }, { "name": "BigBench: Snarks", "description": "BigBench MCQ task: snarks", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_snarks", "is_alpha": false }, { "name": "BigBench: Social Iqa", "description": "BigBench MCQ task: social_iqa", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_social_iqa", "is_alpha": false }, { "name": "BigBench: Social Support", "description": "BigBench MCQ task: social_support", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_social_support", "is_alpha": false }, { "name": "BigBench: Sports Understanding", "description": "BigBench MCQ task: sports_understanding", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_sports_understanding", "is_alpha": false }, { "name": "BigBench: Strange Stories", "description": "BigBench MCQ task: strange_stories", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_strange_stories", "is_alpha": false }, { "name": "BigBench: Strategyqa", "description": "BigBench MCQ task: strategyqa", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_strategyqa", "is_alpha": false }, { "name": "BigBench: Suicide Risk", "description": "BigBench MCQ task: suicide_risk", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_suicide_risk", "is_alpha": false }, { "name": "BigBench: Swahili English Proverbs", "description": "BigBench MCQ task: swahili_english_proverbs", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_swahili_english_proverbs", "is_alpha": false }, { "name": "BigBench: Swedish To German Proverbs", "description": "BigBench MCQ task: swedish_to_german_proverbs", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_swedish_to_german_proverbs", "is_alpha": false }, { "name": "BigBench: Symbol Interpretation", "description": "BigBench MCQ task: symbol_interpretation", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_symbol_interpretation", "is_alpha": false }, { "name": "BigBench: Temporal Sequences", "description": "BigBench MCQ task: temporal_sequences", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_temporal_sequences", "is_alpha": false }, { "name": "BigBench: Timedial", "description": "BigBench MCQ task: timedial", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_timedial", "is_alpha": false }, { "name": "BigBench: Tracking Shuffled Objects", "description": "BigBench MCQ task: tracking_shuffled_objects", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_tracking_shuffled_objects", "is_alpha": false }, { "name": "BigBench: Understanding Fables", "description": "BigBench MCQ task: understanding_fables", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_understanding_fables", "is_alpha": false }, { "name": "BigBench: Undo Permutation", "description": "BigBench MCQ task: undo_permutation", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_undo_permutation", "is_alpha": false }, { "name": "BigBench: Unit Conversion", "description": "BigBench MCQ task: unit_conversion", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_unit_conversion", "is_alpha": false }, { "name": "BigBench: Unit Interpretation", "description": "BigBench MCQ task: unit_interpretation", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_unit_interpretation", "is_alpha": false }, { "name": "BigBench: Vitaminc Fact Verification", "description": "BigBench MCQ task: vitaminc_fact_verification", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_vitaminc_fact_verification", "is_alpha": false }, { "name": "BigBench: What Is The Tao", "description": "BigBench MCQ task: what_is_the_tao", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_what_is_the_tao", "is_alpha": false }, { "name": "BigBench: Which Wiki Edit", "description": "BigBench MCQ task: which_wiki_edit", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_which_wiki_edit", "is_alpha": false }, { "name": "BigBench: Winowhy", "description": "BigBench MCQ task: winowhy", "category": "bigbench", "tags": ["multiple-choice", "reasoning", "bigbench"], "function_name": "bigbench_winowhy", "is_alpha": false }, { "name": "BoolQ", "description": "BoolQ: A Question Answering Dataset for Boolean Reasoning", "category": "core", "tags": ["boolean-reasoning", "question-answering"], "function_name": "boolq", "is_alpha": false }, { "name": "BrowseComp", "description": "A Simple Yet Challenging Benchmark for Browsing Agents - evaluates model performance on browsing-related tasks", "category": "core", "tags": ["browsing", "web", "reasoning", "graded"], "function_name": "browsecomp", "is_alpha": false }, { "name": "COPA", "description": "Choice of Plausible Alternatives for causal reasoning", "category": "glue", "tags": ["multiple-choice", "superglue", "nli", "reasoning"], "function_name": "copa", "is_alpha": false }, { "name": "ChartQAPro", "description": "Comprehensive chart understanding benchmark with 1,948 samples testing vision-language models across 5 question types (Factoid, Multi Choice, Hypothetical, Fact Checking, Conversational) with support for direct, CoT, and PoT prompting strategies", "category": "core", "tags": ["vision", "multi-modal", "charts", "reasoning", "multi-turn"], "function_name": "chartqapro", "is_alpha": false }, { "name": "ClockBench", "description": "Clock benchmark - time-based reasoning tasks", "category": "community", "tags": ["time", "analog", "clock", "reasoning"], "function_name": "clockbench", "is_alpha": false }, { "name": "CoSafe-M2S", "description": "CoSafe m2s - Single turn conversion of the CoSafe dataset", "category": "jailbreak", "tags": ["cosafe", "jailbreak", "m2s"], "function_name": "cosafe_m2s", "is_alpha": false }, { "name": "CommitmentBank", "description": "Natural language inference with commitment", "category": "glue", "tags": ["multiple-choice", "superglue", "nli", "reasoning"], "function_name": "cb", "is_alpha": false }, { "name": "CyBench", "description": "CyBench: Cybersecurity CTF challenges benchmark", "category": "domain-specific", "tags": ["cybersecurity", "ctf", "challenges", "graded"], "function_name": "cybench", "is_alpha": false }, { "name": "DROP", "description": "Reading comprehension benchmark requiring discrete reasoning over paragraphs (arithmetic, counting, sorting)", "category": "core", "tags": ["reading-comprehension", "reasoning", "arithmetic", "counting", "sorting"], "function_name": "drop", "is_alpha": false }, { "name": "DeepResearch Bench", "description": "A comprehensive benchmark for evaluating Deep Research Agents", "category": "core", "tags": ["research", "long-form", "multilingual"], "function_name": "deep_research_bench", "is_alpha": false }, { "name": "DetailBench", "description": "Tests whether LLMs notify users about wrong facts in a text while they are tasked to translate said text", "category": "community", "tags": ["knowledge", "graded", "instruction-following"], "function_name": "detailbench", "is_alpha": false }, { "name": "ETHICS (All Dimensions)", "description": "Aligning AI With Shared Human Values - tests moral reasoning across 5 fundamental dimensions", "category": "ethics-social", "tags": ["multiple-choice", "ethics", "moral-reasoning", "philosophy"], "function_name": "ethics", "is_alpha": false }, { "name": "ETHICS: Commonsense", "description": "Tests everyday moral reasoning and common ethical intuitions", "category": "ethics-social", "tags": ["multiple-choice", "ethics", "moral-reasoning", "commonsense", "ethics"], "function_name": "ethics_commonsense", "is_alpha": false }, { "name": "ETHICS: Deontology", "description": "Tests duty-based ethics and understanding of moral rules", "category": "ethics-social", "tags": ["multiple-choice", "ethics", "moral-reasoning", "deontology", "ethics"], "function_name": "ethics_deontology", "is_alpha": false }, { "name": "ETHICS: Justice", "description": "Tests fairness and impartiality in ethical decision-making", "category": "ethics-social", "tags": ["multiple-choice", "ethics", "moral-reasoning", "justice", "ethics"], "function_name": "ethics_justice", "is_alpha": false }, { "name": "ETHICS: Utilitarianism", "description": "Tests consequence-based ethics and utility maximization", "category": "ethics-social", "tags": ["multiple-choice", "ethics", "moral-reasoning", "utilitarianism", "ethics"], "function_name": "ethics_utilitarianism", "is_alpha": false }, { "name": "ETHICS: Virtue", "description": "Tests character-based ethics and recognition of virtuous behavior", "category": "ethics-social", "tags": ["multiple-choice", "ethics", "moral-reasoning", "virtue", "ethics"], "function_name": "ethics_virtue", "is_alpha": false }, { "name": "Exercism", "description": "Multi-language coding benchmark with real-world programming exercises across Python, Go, JavaScript, Java, and Rust", "category": "core", "tags": ["coding", "multi-language", "execution", "docker"], "function_name": "exercism", "is_alpha": false }, { "name": "Exercism (Go)", "description": "Go coding tasks from the Exercism benchmark", "category": "core", "tags": ["coding", "go", "execution", "docker"], "function_name": "exercism_go", "is_alpha": false }, { "name": "Exercism (Java)", "description": "Java coding tasks from the Exercism benchmark", "category": "core", "tags": ["coding", "java", "execution", "docker"], "function_name": "exercism_java", "is_alpha": false }, { "name": "Exercism (JavaScript)", "description": "JavaScript coding tasks from the Exercism benchmark", "category": "core", "tags": ["coding", "javascript", "execution", "docker"], "function_name": "exercism_javascript", "is_alpha": false }, { "name": "Exercism (Python)", "description": "Python coding tasks from the Exercism benchmark", "category": "core", "tags": ["coding", "python", "execution", "docker"], "function_name": "exercism_python", "is_alpha": false }, { "name": "Exercism (Rust)", "description": "Rust coding tasks from the Exercism benchmark", "category": "core", "tags": ["coding", "rust", "execution", "docker"], "function_name": "exercism_rust", "is_alpha": false }, { "name": "FActScore", "description": "FActScore - a benchmark for fine-grained factuality scoring", "category": "knowledge-qa", "tags": ["factuality", "long-form", "retrieval", "wikipedia"], "function_name": "factscore", "is_alpha": false }, { "name": "GLUE (All Tasks)", "description": "General Language Understanding Evaluation benchmark suite", "category": "glue", "tags": ["multiple-choice", "glue", "nli", "sentiment", "similarity"], "function_name": "glue", "is_alpha": false }, { "name": "GLUE: CoLA", "description": "Corpus of Linguistic Acceptability", "category": "glue", "tags": ["multiple-choice", "glue", "nli"], "function_name": "glue_cola", "is_alpha": false }, { "name": "GLUE: MNLI", "description": "Multi-Genre Natural Language Inference", "category": "glue", "tags": ["multiple-choice", "glue", "nli"], "function_name": "glue_mnli", "is_alpha": false }, { "name": "GLUE: MNLI-MM", "description": "MNLI Mismatched", "category": "glue", "tags": ["multiple-choice", "glue", "nli"], "function_name": "glue_mnli_mismatched", "is_alpha": false }, { "name": "GLUE: MRPC", "description": "Microsoft Research Paraphrase Corpus", "category": "glue", "tags": ["multiple-choice", "glue", "nli"], "function_name": "glue_mrpc", "is_alpha": false }, { "name": "GLUE: QNLI", "description": "Question Natural Language Inference", "category": "glue", "tags": ["multiple-choice", "glue", "nli"], "function_name": "glue_qnli", "is_alpha": false }, { "name": "GLUE: QQP", "description": "Quora Question Pairs", "category": "glue", "tags": ["multiple-choice", "glue", "nli"], "function_name": "glue_qqp", "is_alpha": false }, { "name": "GLUE: RTE", "description": "Recognizing Textual Entailment", "category": "glue", "tags": ["multiple-choice", "glue", "nli"], "function_name": "glue_rte", "is_alpha": false }, { "name": "GLUE: SST-2", "description": "Stanford Sentiment Treebank", "category": "glue", "tags": ["multiple-choice", "glue", "nli"], "function_name": "glue_sst2", "is_alpha": false }, { "name": "GLUE: STS-B", "description": "Semantic Textual Similarity Benchmark", "category": "glue", "tags": ["multiple-choice", "glue", "nli"], "function_name": "glue_stsb", "is_alpha": false }, { "name": "GLUE: WNLI", "description": "Winograd Natural Language Inference", "category": "glue", "tags": ["multiple-choice", "glue", "nli"], "function_name": "glue_wnli", "is_alpha": false }, { "name": "GMCQ", "description": "GitHub Multiple Choice Questions", "category": "core", "tags": ["code-understanding"], "function_name": "rootly_gmcq", "is_alpha": false }, { "name": "GPQA", "description": "Graduate-level science questions (multiple choice) across physics, chemistry, and biology", "category": "core", "tags": ["multiple-choice", "science", "graduate-level", "reasoning"], "function_name": "gpqa", "is_alpha": false }, { "name": "GPQA Diamond", "description": "Graduate-level Google-Proof Q&A in biology, chemistry, and physics", "category": "core", "tags": ["multiple-choice", "science", "graduate-level"], "function_name": "gpqa_diamond", "is_alpha": false }, { "name": "GPT-OSS AIME 2025", "description": "An exact replica of the aime25 eval from OpenAI's GPT-OSS release", "category": "core", "tags": ["math", "competition", "aime", "2025"], "function_name": "gpt_oss_aime25", "is_alpha": false }, { "name": "GSM-Plus", "description": "GSM-Plus robustness benchmark with perturbations to test math reasoning stability", "category": "core", "tags": ["math", "reasoning", "robustness", "word-problems"], "function_name": "gsm_plus", "is_alpha": false }, { "name": "GSM-Plus Mini", "description": "GSM-Plus smaller test set (2400 samples)", "category": "core", "tags": ["math", "reasoning", "robustness", "word-problems"], "function_name": "gsm_plus_mini", "is_alpha": false }, { "name": "GSM8K", "description": "Grade School Math 8K - 8,500 diverse grade school math word problems requiring multi-step reasoning", "category": "core", "tags": ["math", "reasoning", "chain-of-thought", "word-problems"], "function_name": "gsm8k", "is_alpha": false }, { "name": "Global-MMLU (42 Languages)", "description": "Culturally adapted multilingual MMLU with 42 languages", "category": "core", "tags": ["multiple-choice", "multilingual", "cultural-sensitivity", "mmlu"], "function_name": "global_mmlu", "is_alpha": false }, { "name": "Global-MMLU: Amharic", "description": "Global-MMLU culturally adapted MMLU for Amharic (am)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_amharic", "is_alpha": false }, { "name": "Global-MMLU: Arabic", "description": "Global-MMLU culturally adapted MMLU for Arabic (ar)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_arabic", "is_alpha": false }, { "name": "Global-MMLU: Bengali", "description": "Global-MMLU culturally adapted MMLU for Bengali (bn)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_bengali", "is_alpha": false }, { "name": "Global-MMLU: Chichewa", "description": "Global-MMLU culturally adapted MMLU for Chichewa (ny)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_chichewa", "is_alpha": false }, { "name": "Global-MMLU: Chinese", "description": "Global-MMLU culturally adapted MMLU for Chinese (zh)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_chinese", "is_alpha": false }, { "name": "Global-MMLU: Czech", "description": "Global-MMLU culturally adapted MMLU for Czech (cs)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_czech", "is_alpha": false }, { "name": "Global-MMLU: Dutch", "description": "Global-MMLU culturally adapted MMLU for Dutch (nl)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_dutch", "is_alpha": false }, { "name": "Global-MMLU: English", "description": "Global-MMLU culturally adapted MMLU for English (en)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_english", "is_alpha": false }, { "name": "Global-MMLU: Filipino", "description": "Global-MMLU culturally adapted MMLU for Filipino (fil)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_filipino", "is_alpha": false }, { "name": "Global-MMLU: French", "description": "Global-MMLU culturally adapted MMLU for French (fr)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_french", "is_alpha": false }, { "name": "Global-MMLU: German", "description": "Global-MMLU culturally adapted MMLU for German (de)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_german", "is_alpha": false }, { "name": "Global-MMLU: Greek", "description": "Global-MMLU culturally adapted MMLU for Greek (el)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_greek", "is_alpha": false }, { "name": "Global-MMLU: Hausa", "description": "Global-MMLU culturally adapted MMLU for Hausa (ha)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_hausa", "is_alpha": false }, { "name": "Global-MMLU: Hebrew", "description": "Global-MMLU culturally adapted MMLU for Hebrew (he)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_hebrew", "is_alpha": false }, { "name": "Global-MMLU: Hindi", "description": "Global-MMLU culturally adapted MMLU for Hindi (hi)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_hindi", "is_alpha": false }, { "name": "Global-MMLU: Igbo", "description": "Global-MMLU culturally adapted MMLU for Igbo (ig)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_igbo", "is_alpha": false }, { "name": "Global-MMLU: Indonesian", "description": "Global-MMLU culturally adapted MMLU for Indonesian (id)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_indonesian", "is_alpha": false }, { "name": "Global-MMLU: Italian", "description": "Global-MMLU culturally adapted MMLU for Italian (it)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_italian", "is_alpha": false }, { "name": "Global-MMLU: Japanese", "description": "Global-MMLU culturally adapted MMLU for Japanese (ja)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_japanese", "is_alpha": false }, { "name": "Global-MMLU: Korean", "description": "Global-MMLU culturally adapted MMLU for Korean (ko)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_korean", "is_alpha": false }, { "name": "Global-MMLU: Kyrgyz", "description": "Global-MMLU culturally adapted MMLU for Kyrgyz (ky)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_kyrgyz", "is_alpha": false }, { "name": "Global-MMLU: Lithuanian", "description": "Global-MMLU culturally adapted MMLU for Lithuanian (lt)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_lithuanian", "is_alpha": false }, { "name": "Global-MMLU: Malagasy", "description": "Global-MMLU culturally adapted MMLU for Malagasy (mg)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_malagasy", "is_alpha": false }, { "name": "Global-MMLU: Malay", "description": "Global-MMLU culturally adapted MMLU for Malay (ms)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_malay", "is_alpha": false }, { "name": "Global-MMLU: Nepali", "description": "Global-MMLU culturally adapted MMLU for Nepali (ne)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_nepali", "is_alpha": false }, { "name": "Global-MMLU: Persian", "description": "Global-MMLU culturally adapted MMLU for Persian (fa)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_persian", "is_alpha": false }, { "name": "Global-MMLU: Polish", "description": "Global-MMLU culturally adapted MMLU for Polish (pl)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_polish", "is_alpha": false }, { "name": "Global-MMLU: Portuguese", "description": "Global-MMLU culturally adapted MMLU for Portuguese (pt)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_portuguese", "is_alpha": false }, { "name": "Global-MMLU: Romanian", "description": "Global-MMLU culturally adapted MMLU for Romanian (ro)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_romanian", "is_alpha": false }, { "name": "Global-MMLU: Russian", "description": "Global-MMLU culturally adapted MMLU for Russian (ru)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_russian", "is_alpha": false }, { "name": "Global-MMLU: Serbian", "description": "Global-MMLU culturally adapted MMLU for Serbian (sr)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_serbian", "is_alpha": false }, { "name": "Global-MMLU: Shona", "description": "Global-MMLU culturally adapted MMLU for Shona (sn)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_shona", "is_alpha": false }, { "name": "Global-MMLU: Sinhala", "description": "Global-MMLU culturally adapted MMLU for Sinhala (si)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_sinhala", "is_alpha": false }, { "name": "Global-MMLU: Somali", "description": "Global-MMLU culturally adapted MMLU for Somali (so)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_somali", "is_alpha": false }, { "name": "Global-MMLU: Spanish", "description": "Global-MMLU culturally adapted MMLU for Spanish (es)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_spanish", "is_alpha": false }, { "name": "Global-MMLU: Swahili", "description": "Global-MMLU culturally adapted MMLU for Swahili (sw)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_swahili", "is_alpha": false }, { "name": "Global-MMLU: Swedish", "description": "Global-MMLU culturally adapted MMLU for Swedish (sv)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_swedish", "is_alpha": false }, { "name": "Global-MMLU: Telugu", "description": "Global-MMLU culturally adapted MMLU for Telugu (te)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_telugu", "is_alpha": false }, { "name": "Global-MMLU: Turkish", "description": "Global-MMLU culturally adapted MMLU for Turkish (tr)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_turkish", "is_alpha": false }, { "name": "Global-MMLU: Ukrainian", "description": "Global-MMLU culturally adapted MMLU for Ukrainian (uk)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_ukrainian", "is_alpha": false }, { "name": "Global-MMLU: Vietnamese", "description": "Global-MMLU culturally adapted MMLU for Vietnamese (vi)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_vietnamese", "is_alpha": false }, { "name": "Global-MMLU: Yoruba", "description": "Global-MMLU culturally adapted MMLU for Yoruba (yo)", "category": "global-mmlu", "tags": ["multiple-choice", "multilingual", "cultural-adaptation", "global-mmlu"], "function_name": "global_mmlu_yoruba", "is_alpha": false }, { "name": "GraphWalks", "description": "Multi-hop reasoning on graphs - both BFS and parent finding tasks", "category": "core", "tags": ["long-context", "graphs", "reasoning", "alpha"], "function_name": "graphwalks", "is_alpha": true }, { "name": "GraphWalks BFS", "description": "Multi-hop reasoning on graphs - BFS traversal tasks only", "category": "core", "tags": ["long-context", "graphs", "reasoning", "bfs", "alpha"], "function_name": "graphwalks_bfs", "is_alpha": true }, { "name": "GraphWalks Parents", "description": "Multi-hop reasoning on graphs - parent finding tasks only", "category": "core", "tags": ["long-context", "graphs", "reasoning", "parents", "alpha"], "function_name": "graphwalks_parents", "is_alpha": true }, { "name": "HMMT Feb 2023", "description": "Harvard-MIT Mathematics Tournament February 2023", "category": "math", "tags": ["math", "competition", "hmmt", "2023"], "function_name": "hmmt_feb_2023", "is_alpha": false }, { "name": "HMMT Feb 2024", "description": "Harvard-MIT Mathematics Tournament February 2024", "category": "math", "tags": ["math", "competition", "hmmt", "2024"], "function_name": "hmmt_feb_2024", "is_alpha": false }, { "name": "HMMT Feb 2025", "description": "Harvard-MIT Mathematics Tournament February 2025", "category": "math", "tags": ["math", "competition", "hmmt", "2025"], "function_name": "hmmt_feb_2025", "is_alpha": false }, { "name": "HeadQA", "description": "Spanish healthcare specialization exam questions (Spanish and English)", "category": "core", "tags": ["multiple-choice", "medical", "healthcare", "multilingual"], "function_name": "headqa", "is_alpha": false }, { "name": "HeadQA (English)", "description": "Spanish healthcare specialization exam questions in English", "category": "core", "tags": ["multiple-choice", "medical", "healthcare", "english"], "function_name": "headqa_en", "is_alpha": false }, { "name": "HeadQA (Spanish)", "description": "Spanish healthcare specialization exam questions in Spanish", "category": "core", "tags": ["multiple-choice", "medical", "healthcare", "spanish"], "function_name": "headqa_es", "is_alpha": false }, { "name": "HealthBench", "description": "Medical dialogue evaluation using physician-created rubrics for assessing healthcare conversations", "category": "core", "tags": ["medical", "dialogue", "graded", "rubric-based"], "function_name": "healthbench", "is_alpha": false }, { "name": "HealthBench Consensus", "description": "Medical dialogue cases with strong physician consensus on appropriate responses", "category": "core", "tags": ["medical", "dialogue", "graded", "rubric-based", "consensus"], "function_name": "healthbench_consensus", "is_alpha": false }, { "name": "HealthBench Hard", "description": "Most challenging medical dialogue cases from HealthBench requiring nuanced medical knowledge", "category": "core", "tags": ["medical", "dialogue", "graded", "rubric-based", "hard"], "function_name": "healthbench_hard", "is_alpha": false }, { "name": "HellaSwag", "description": "Adversarially-filtered sentence completion benchmark for commonsense reasoning", "category": "core", "tags": ["multiple-choice", "commonsense-reasoning", "sentence-completion"], "function_name": "hellaswag", "is_alpha": false }, { "name": "HumanEval", "description": "Code generation benchmark with 164 programming problems", "category": "core", "tags": ["coding", "generation", "execution"], "function_name": "humaneval", "is_alpha": false }, { "name": "Humanity's Last Exam", "description": "Multi-modal benchmark at the frontier of human knowledge - 2,500 questions across mathematics, humanities, and natural sciences designed by subject-matter experts globally", "category": "core", "tags": ["knowledge", "reasoning", "multi-modal", "graded", "frontier"], "function_name": "hle", "is_alpha": false }, { "name": "Humanity's Last Exam (Text-Only)", "description": "Text-only variant of HLE with multi-modal questions filtered out - evaluates models without vision capabilities on text-based questions from the frontier of human knowledge", "category": "core", "tags": ["knowledge", "reasoning", "text-only", "graded", "frontier"], "function_name": "hle_text", "is_alpha": false }, { "name": "IFBench", "description": "Out-of-distribution instruction following with strict and loose scoring over challenging constraints", "category": "core", "tags": ["instruction-following", "constraints", "formatting", "ood"], "function_name": "ifbench", "is_alpha": false }, { "name": "Instruction Following", "description": "Tests ability to follow specific formatting and content constraints with both strict and loose evaluation metrics", "category": "core", "tags": ["instruction-following", "constraints", "formatting"], "function_name": "ifeval", "is_alpha": false }, { "name": "JSONSchemaBench", "description": "JSON Schema generation benchmark with ~10K real-world schemas from GitHub, Kubernetes, and other sources for evaluating constrained decoding", "category": "core", "tags": ["json", "jsonschema", "generation", "constrained-decoding"], "function_name": "jsonschemabench", "is_alpha": false }, { "name": "LegalSupport", "description": "Legal citation support identification - identify which citation provides stronger support for a legal argument", "category": "domain-specific", "tags": ["multiple-choice", "legal", "reasoning", "citation-analysis"], "function_name": "legalsupport", "is_alpha": false }, { "name": "LiveMCPBench", "description": "Benchmark for evaluating LLM agents on real-world tasks using the Model Context Protocol (MCP) - 95 tasks across different categories", "category": "core", "tags": ["mcp", "agents", "real-world", "tools", "graded"], "function_name": "livemcpbench", "is_alpha": false }, { "name": "LogiQA", "description": "Logical reasoning dataset from Chinese civil service exam questions - tests deductive reasoning skills", "category": "knowledge-qa", "tags": ["multiple-choice", "logical-reasoning", "deduction", "critical-thinking"], "function_name": "logiqa", "is_alpha": false }, { "name": "MATH", "description": "Measuring Mathematical Problem Solving - 5000 competition math problems across 7 subjects and 5 difficulty levels", "category": "core", "tags": ["math", "problem-solving", "reasoning", "competition", "graded"], "function_name": "math", "is_alpha": false }, { "name": "MATH-500", "description": "500-problem subset of MATH dataset for faster evaluation of mathematical problem solving", "category": "core", "tags": ["math", "problem-solving", "reasoning", "competition", "graded", "subset"], "function_name": "math_500", "is_alpha": false }, { "name": "MBPP", "description": "Mostly Basic Python Problems — code generation tasks with unit test verification", "category": "core", "tags": ["code", "generation", "sandbox", "reasoning"], "function_name": "mbpp", "is_alpha": false }, { "name": "MGSM", "description": "Multilingual Grade School Math benchmark across 11 languages for testing mathematical reasoning", "category": "core", "tags": ["math", "multilingual", "reasoning", "chain-of-thought"], "function_name": "mgsm", "is_alpha": false }, { "name": "MGSM (Bengali)", "description": "Multilingual Grade School Math in Bengali", "category": "math", "tags": ["math", "reasoning", "multilingual", "mgsm"], "function_name": "mgsm_bn", "is_alpha": false }, { "name": "MGSM (Chinese)", "description": "Multilingual Grade School Math in Chinese", "category": "math", "tags": ["math", "reasoning", "multilingual", "mgsm"], "function_name": "mgsm_zh", "is_alpha": false }, { "name": "MGSM (French)", "description": "Multilingual Grade School Math in French", "category": "math", "tags": ["math", "reasoning", "multilingual", "mgsm"], "function_name": "mgsm_fr", "is_alpha": false }, { "name": "MGSM (German)", "description": "Multilingual Grade School Math in German", "category": "math", "tags": ["math", "reasoning", "multilingual", "mgsm"], "function_name": "mgsm_de", "is_alpha": false }, { "name": "MGSM (Japanese)", "description": "Multilingual Grade School Math in Japanese", "category": "math", "tags": ["math", "reasoning", "multilingual", "mgsm"], "function_name": "mgsm_ja", "is_alpha": false }, { "name": "MGSM (Russian)", "description": "Multilingual Grade School Math in Russian", "category": "math", "tags": ["math", "reasoning", "multilingual", "mgsm"], "function_name": "mgsm_ru", "is_alpha": false }, { "name": "MGSM (Spanish)", "description": "Multilingual Grade School Math in Spanish", "category": "math", "tags": ["math", "reasoning", "multilingual", "mgsm"], "function_name": "mgsm_es", "is_alpha": false }, { "name": "MGSM (Swahili)", "description": "Multilingual Grade School Math in Swahili", "category": "math", "tags": ["math", "reasoning", "multilingual", "mgsm"], "function_name": "mgsm_sw", "is_alpha": false }, { "name": "MGSM (Telugu)", "description": "Multilingual Grade School Math in Telugu", "category": "math", "tags": ["math", "reasoning", "multilingual", "mgsm"], "function_name": "mgsm_te", "is_alpha": false }, { "name": "MGSM (Thai)", "description": "Multilingual Grade School Math in Thai", "category": "math", "tags": ["math", "reasoning", "multilingual", "mgsm"], "function_name": "mgsm_th", "is_alpha": false }, { "name": "MGSM English", "description": "Grade school math problems in English for testing mathematical reasoning", "category": "core", "tags": ["math", "english", "reasoning", "chain-of-thought"], "function_name": "mgsm_en", "is_alpha": false }, { "name": "MGSM Latin Script", "description": "Grade school math problems in Latin script languages (German, English, Spanish, French, Swahili)", "category": "core", "tags": ["math", "multilingual", "latin-script", "reasoning", "chain-of-thought"], "function_name": "mgsm_latin", "is_alpha": false }, { "name": "MGSM Non-Latin Script", "description": "Grade school math problems in non-Latin script languages (Bengali, Japanese, Russian, Telugu, Thai, Chinese)", "category": "core", "tags": ["math", "multilingual", "non-latin-script", "reasoning", "chain-of-thought"], "function_name": "mgsm_non_latin", "is_alpha": false }, { "name": "MHJ-M2S", "description": "MHJ-M2S - Single turn conversion of the MHJ dataset", "category": "jailbreak", "tags": ["mhj", "jailbreak", "m2s"], "function_name": "mhj_m2s", "is_alpha": false }, { "name": "MM-Vet v2", "description": "Challenging benchmark evaluating large multimodal models for integrated capabilities using GPT-4 judge with 5-run grading average (configurable via --task-args num_grading_attempts=N)", "category": "multimodal", "tags": ["multimodal", "vision", "llm-judge"], "function_name": "mmvetv2", "is_alpha": false }, { "name": "MMLU (cais/mmlu)", "description": "Massive Multitask Language Understanding - 57 academic subjects from the cais/mmlu dataset. Only supports English (EN-US).", "category": "core", "tags": ["multiple-choice", "knowledge", "reasoning", "multitask"], "function_name": "mmlu", "is_alpha": false }, { "name": "MMLU Pro (TIGER-Lab)", "description": "Enhanced version of MMLU with more challenging, reasoning-focused questions.", "category": "core", "tags": ["multiple-choice", "knowledge", "reasoning", "multitask"], "function_name": "mmlu_pro", "is_alpha": false }, { "name": "MMLU-Redux", "description": "Manually re-annotated subset of 5,700 MMLU questions addressing annotation errors in the original dataset.", "category": "core", "tags": ["multiple-choice", "knowledge", "reasoning", "multitask"], "function_name": "mmlu_redux", "is_alpha": false }, { "name": "MMMLU (Arabic)", "description": "MMLU in Arabic (AR_XY)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_ar_xy", "is_alpha": false }, { "name": "MMMLU (Bengali)", "description": "MMLU in Bengali (BN_BD)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_bn_bd", "is_alpha": false }, { "name": "MMMLU (Chinese)", "description": "MMLU in Chinese (ZH_CN)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_zh_cn", "is_alpha": false }, { "name": "MMMLU (French)", "description": "MMLU in French (FR_FR)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_fr_fr", "is_alpha": false }, { "name": "MMMLU (German)", "description": "MMLU in German (DE_DE)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_de_de", "is_alpha": false }, { "name": "MMMLU (Hindi)", "description": "MMLU in Hindi (HI_IN)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_hi_in", "is_alpha": false }, { "name": "MMMLU (Indonesian)", "description": "MMLU in Indonesian (ID_ID)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_id_id", "is_alpha": false }, { "name": "MMMLU (Italian)", "description": "MMLU in Italian (IT_IT)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_it_it", "is_alpha": false }, { "name": "MMMLU (Japanese)", "description": "MMLU in Japanese (JA_JP)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_ja_jp", "is_alpha": false }, { "name": "MMMLU (Korean)", "description": "MMLU in Korean (KO_KR)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_ko_kr", "is_alpha": false }, { "name": "MMMLU (Portuguese)", "description": "MMLU in Portuguese Brazil (PT_BR)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_pt_br", "is_alpha": false }, { "name": "MMMLU (Spanish)", "description": "MMLU in Spanish Latin America (ES_LA)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_es_la", "is_alpha": false }, { "name": "MMMLU (Swahili)", "description": "MMLU in Swahili (SW_KE)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_sw_ke", "is_alpha": false }, { "name": "MMMLU (Yoruba)", "description": "MMLU in Yoruba (YO_NG)", "category": "core", "tags": ["multiple-choice", "knowledge", "multilingual", "mmmlu"], "function_name": "mmmlu_yo_ng", "is_alpha": false }, { "name": "MMMLU (openai/MMMLU)", "description": "MMLU translated to 15 languages.", "category": "core", "tags": ["multiple-choice", "knowledge", "reasoning", "multitask"], "function_name": "mmmlu", "is_alpha": false }, { "name": "MMMU", "description": "Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark with 11.5K questions across 30 subjects from college exams, quizzes, and textbooks", "category": "core", "tags": ["multimodal", "multiple-choice", "reasoning", "college-level", "images"], "function_name": "mmmu", "is_alpha": false }, { "name": "MMMU Accounting", "description": "MMMU Accounting subset focusing on accounting principles and practices", "category": "core", "tags": ["multimodal", "multiple-choice", "accounting", "business", "images"], "function_name": "mmmu_accounting", "is_alpha": false }, { "name": "MMMU Agriculture", "description": "MMMU Agriculture subset focusing on agricultural sciences and practices", "category": "core", "tags": ["multimodal", "multiple-choice", "agriculture", "science", "images"], "function_name": "mmmu_agriculture", "is_alpha": false }, { "name": "MMMU Architecture and Engineering", "description": "MMMU Architecture and Engineering subset focusing on engineering design and architecture", "category": "core", "tags": ["multimodal", "multiple-choice", "architecture", "engineering", "design", "images"], "function_name": "mmmu_architecture_and_engineering", "is_alpha": false }, { "name": "MMMU Art", "description": "MMMU Art subset focusing on art and visual design questions", "category": "core", "tags": ["multimodal", "multiple-choice", "art", "visual-design", "images"], "function_name": "mmmu_art", "is_alpha": false }, { "name": "MMMU Art Theory", "description": "MMMU Art Theory subset focusing on art history and theoretical concepts", "category": "core", "tags": ["multimodal", "multiple-choice", "art", "theory", "history", "images"], "function_name": "mmmu_art_theory", "is_alpha": false }, { "name": "MMMU Basic Medical Science", "description": "MMMU Basic Medical Science subset focusing on fundamental medical knowledge", "category": "core", "tags": ["multimodal", "multiple-choice", "medicine", "science", "health", "images"], "function_name": "mmmu_basic_medical_science", "is_alpha": false }, { "name": "MMMU Biology", "description": "MMMU Biology subset focusing on biological sciences", "category": "core", "tags": ["multimodal", "multiple-choice", "biology", "science", "images"], "function_name": "mmmu_biology", "is_alpha": false }, { "name": "MMMU Chemistry", "description": "MMMU Chemistry subset focusing on chemical sciences", "category": "core", "tags": ["multimodal", "multiple-choice", "chemistry", "science", "images"], "function_name": "mmmu_chemistry", "is_alpha": false }, { "name": "MMMU Clinical Medicine", "description": "MMMU Clinical Medicine subset focusing on clinical medical practice", "category": "core", "tags": ["multimodal", "multiple-choice", "medicine", "clinical", "health", "images"], "function_name": "mmmu_clinical_medicine", "is_alpha": false }, { "name": "MMMU Design", "description": "MMMU Design subset focusing on design principles and practices", "category": "core", "tags": ["multimodal", "multiple-choice", "design", "visual", "creative", "images"], "function_name": "mmmu_design", "is_alpha": false }, { "name": "MMMU Diagnostics and Laboratory Medicine", "description": "MMMU Diagnostics and Laboratory Medicine subset focusing on medical diagnostics", "category": "core", "tags": ["multimodal", "multiple-choice", "medicine", "diagnostics", "laboratory", "images"], "function_name": "mmmu_diagnostics_and_laboratory_medicine", "is_alpha": false }, { "name": "MMMU Electronics", "description": "MMMU Electronics subset focusing on electronic systems and circuits", "category": "core", "tags": ["multimodal", "multiple-choice", "electronics", "engineering", "technology", "images"], "function_name": "mmmu_electronics", "is_alpha": false }, { "name": "MMMU Energy and Power", "description": "MMMU Energy and Power subset focusing on energy systems and power generation", "category": "core", "tags": ["multimodal", "multiple-choice", "energy", "power", "engineering", "images"], "function_name": "mmmu_energy_and_power", "is_alpha": false }, { "name": "MMMU Finance", "description": "MMMU Finance subset focusing on financial concepts and analysis", "category": "core", "tags": ["multimodal", "multiple-choice", "finance", "business", "economics", "images"], "function_name": "mmmu_finance", "is_alpha": false }, { "name": "MMMU Geography", "description": "MMMU Geography subset focusing on geographical knowledge and analysis", "category": "core", "tags": ["multimodal", "multiple-choice", "geography", "earth-science", "spatial", "images"], "function_name": "mmmu_geography", "is_alpha": false }, { "name": "MMMU History", "description": "MMMU History subset focusing on historical knowledge and analysis", "category": "core", "tags": ["multimodal", "multiple-choice", "history", "humanities", "culture", "images"], "function_name": "mmmu_history", "is_alpha": false }, { "name": "MMMU Literature", "description": "MMMU Literature subset focusing on literary analysis and knowledge", "category": "core", "tags": ["multimodal", "multiple-choice", "literature", "humanities", "language", "images"], "function_name": "mmmu_literature", "is_alpha": false }, { "name": "MMMU MCQ", "description": "MMMU MCQ subset focusing on multiple choice questions", "category": "core", "tags": ["multimodal", "multiple-choice", "images"], "function_name": "mmmu_mcq", "is_alpha": false }, { "name": "MMMU Management", "description": "MMMU Management subset focusing on management principles and practices", "category": "core", "tags": ["multimodal", "multiple-choice", "management", "business", "leadership", "images"], "function_name": "mmmu_manage", "is_alpha": false }, { "name": "MMMU Marketing", "description": "MMMU Marketing subset focusing on marketing strategies and concepts", "category": "core", "tags": ["multimodal", "multiple-choice", "marketing", "business", "communication", "images"], "function_name": "mmmu_marketing", "is_alpha": false }, { "name": "MMMU Materials", "description": "MMMU Materials subset focusing on materials science and engineering", "category": "core", "tags": ["multimodal", "multiple-choice", "materials", "science", "engineering", "images"], "function_name": "mmmu_materials", "is_alpha": false }, { "name": "MMMU Math", "description": "MMMU Mathematics subset focusing on mathematical reasoning", "category": "math", "tags": ["multimodal", "multiple-choice", "mathematics", "reasoning", "images"], "function_name": "mmmu_math", "is_alpha": false }, { "name": "MMMU Mechanical Engineering", "description": "MMMU Mechanical Engineering subset focusing on mechanical systems and design", "category": "core", "tags": ["multimodal", "multiple-choice", "mechanical", "engineering", "design", "images"], "function_name": "mmmu_mechanical_engineering", "is_alpha": false }, { "name": "MMMU Music", "description": "MMMU Music subset focusing on music theory and analysis", "category": "core", "tags": ["multimodal", "multiple-choice", "music", "arts", "theory", "images"], "function_name": "mmmu_music", "is_alpha": false }, { "name": "MMMU Open", "description": "MMMU Open subset focusing on open-ended questions", "category": "core", "tags": ["multimodal", "open-ended", "images"], "function_name": "mmmu_open", "is_alpha": false }, { "name": "MMMU Pharmacy", "description": "MMMU Pharmacy subset focusing on pharmaceutical sciences and practice", "category": "core", "tags": ["multimodal", "multiple-choice", "pharmacy", "medicine", "health", "images"], "function_name": "mmmu_pharmacy", "is_alpha": false }, { "name": "MMMU Physics", "description": "MMMU Physics subset focusing on physics and physical sciences", "category": "core", "tags": ["multimodal", "multiple-choice", "physics", "science", "images"], "function_name": "mmmu_physics", "is_alpha": false }, { "name": "MMMU Public Health", "description": "MMMU Public Health subset focusing on public health concepts and practices", "category": "core", "tags": ["multimodal", "multiple-choice", "public-health", "health", "population", "images"], "function_name": "mmmu_public_health", "is_alpha": false }, { "name": "MMMU Sociology", "description": "MMMU Sociology subset focusing on sociological concepts and analysis", "category": "core", "tags": ["multimodal", "multiple-choice", "sociology", "social-science", "society", "images"], "function_name": "mmmu_sociology", "is_alpha": false }, { "name": "MMMU-Pro", "description": "Enhanced multimodal MMMU-Pro benchmark with multiple-choice across many options and images", "category": "core", "tags": ["multimodal", "multiple-choice", "reasoning", "images", "mmmu-pro"], "function_name": "mmmu_pro", "is_alpha": false }, { "name": "MMMU-Pro (Vision)", "description": "MMMU-Pro vision subset with images and multiple-choice questions", "category": "core", "tags": ["multimodal", "vision", "multiple-choice", "images", "mmmu-pro"], "function_name": "mmmu_pro_vision", "is_alpha": false }, { "name": "MMStar", "description": "MMStar benchmark for measuring multi-modal gain and leakage via coordinated vision and text ablations", "category": "core", "tags": ["vision", "multi-modal", "leakage", "perception", "reasoning"], "function_name": "mmstar", "is_alpha": false }, { "name": "MathQA", "description": "Mathematical word problems with multiple-choice answers and solution rationales", "category": "knowledge-qa", "tags": ["multiple-choice", "mathematics", "word-problems", "reasoning"], "function_name": "math_qa", "is_alpha": false }, { "name": "MathVista", "description": "Mathematical Reasoning in Visual Contexts - 6.1K multimodal math problems requiring fine-grained visual understanding across diverse visual contexts", "category": "core", "tags": ["multimodal", "math", "reasoning", "visual", "images", "math-vision"], "function_name": "mathvista", "is_alpha": false }, { "name": "MedMCQA", "description": "Medical multiple-choice questions from Indian medical entrance exams (AIIMS & NEET PG)", "category": "core", "tags": ["multiple-choice", "medical", "healthcare", "medicine"], "function_name": "medmcqa", "is_alpha": false }, { "name": "MedQA", "description": "US Medical Licensing Exam (USMLE) questions for medical reasoning", "category": "core", "tags": ["multiple-choice", "medical", "healthcare", "medicine", "clinical"], "function_name": "medqa", "is_alpha": false }, { "name": "MockAIME (2024)", "description": "Otis Mock AIME - a benchmark from the OTIS Mock AIME 2024 exam", "category": "math", "tags": ["aime", "problem-solving", "math", "2024"], "function_name": "otis_mock_aime_2024", "is_alpha": false }, { "name": "MockAIME (2024-2025)", "description": "Otis Mock AIME - a benchmark from the OTIS Mock AIME 2024-2025 exams", "category": "math", "tags": ["aime", "problem-solving", "math", "2024-2025"], "function_name": "otis_mock_aime", "is_alpha": false }, { "name": "MockAIME (2025)", "description": "Otis Mock AIME - a benchmark from the OTIS Mock AIME 2025 exams", "category": "math", "tags": ["aime", "problem-solving", "math", "2025"], "function_name": "otis_mock_aime_2025", "is_alpha": false }, { "name": "MuSR", "description": "Testing the Limits of Chain-of-thought with Multistep Soft Reasoning - includes murder mysteries, object placements, and team allocation tasks", "category": "core", "tags": ["multiple-choice", "reasoning", "commonsense", "chain-of-thought"], "function_name": "musr", "is_alpha": false }, { "name": "MuSR Murder Mysteries", "description": "MuSR murder mystery scenarios - who is the most likely murderer?", "category": "core", "tags": ["multiple-choice", "reasoning", "commonsense", "chain-of-thought", "murder-mysteries"], "function_name": "musr_murder_mysteries", "is_alpha": false }, { "name": "MuSR Object Placements", "description": "MuSR object placement reasoning - where would someone look for an object?", "category": "core", "tags": ["multiple-choice", "reasoning", "commonsense", "chain-of-thought", "object-placements"], "function_name": "musr_object_placements", "is_alpha": false }, { "name": "MuSR Team Allocation", "description": "MuSR team allocation problems - how to allocate people to tasks efficiently?", "category": "core", "tags": ["multiple-choice", "reasoning", "commonsense", "chain-of-thought", "team-allocation"], "function_name": "musr_team_allocation", "is_alpha": false }, { "name": "MultiChallenge", "description": "Multi-turn conversational tasks requiring reasoning, instruction retention, and coherence.", "category": "core", "tags": ["multi-turn", "reasoning", "chat", "judged"], "function_name": "multichallenge", "is_alpha": false }, { "name": "MultiRC", "description": "Multi-Sentence Reading Comprehension", "category": "glue", "tags": ["multiple-choice", "superglue", "nli", "reasoning"], "function_name": "multirc", "is_alpha": false }, { "name": "Natural Questions", "description": "Google's open-domain QA dataset with real Google queries answered from Wikipedia", "category": "core", "tags": ["question-answering", "factuality", "open-domain"], "function_name": "natural_questions", "is_alpha": false }, { "name": "OCRBench v2", "description": "Visual text localization and reasoning benchmark across 31 diverse OCR and document understanding scenarios", "category": "core", "tags": ["multimodal", "vision-language", "ocr", "document-understanding", "images", "vision"], "function_name": "ocrbenchv2", "is_alpha": false }, { "name": "OpenAI MRCR (2 Needles)", "description": "Memory-Recall with Contextual Retrieval - long-context evaluation that measures recall of 2 needles across million-token contexts", "category": "core", "tags": ["long-context", "retrieval", "needle", "sequence-matching"], "function_name": "openai_mrcr_2n", "is_alpha": false }, { "name": "OpenAI MRCR (4 Needles)", "description": "Memory-Recall with Contextual Retrieval - long-context evaluation that measures recall of 4 needles across million-token contexts", "category": "core", "tags": ["long-context", "retrieval", "needle", "sequence-matching"], "function_name": "openai_mrcr_4n", "is_alpha": false }, { "name": "OpenAI MRCR (8 Needles)", "description": "Memory-Recall with Contextual Retrieval - long-context evaluation that measures recall of 8 needles across million-token contexts", "category": "core", "tags": ["long-context", "retrieval", "needle", "sequence-matching"], "function_name": "openai_mrcr_8n", "is_alpha": false }, { "name": "OpenAI MRCR (Full)", "description": "Memory-Recall with Contextual Retrieval - long-context evaluation that measures recall of 2, 4, and 8 needles across million-token contexts", "category": "core", "tags": ["long-context", "retrieval", "needle", "sequence-matching"], "function_name": "openai_mrcr", "is_alpha": false }, { "name": "OpenBookQA", "description": "Elementary-level science questions probing understanding of core facts", "category": "core", "tags": ["multiple-choice", "science", "elementary", "open-book"], "function_name": "openbookqa", "is_alpha": false }, { "name": "PIQA", "description": "Physical Interaction Question Answering - commonsense about physical situations", "category": "core", "tags": ["multiple-choice", "commonsense-reasoning", "physical-reasoning"], "function_name": "piqa", "is_alpha": false }, { "name": "PROST", "description": "Physical Reasoning about Objects through Space and Time", "category": "core", "tags": ["multiple-choice", "commonsense-reasoning", "physical-reasoning"], "function_name": "prost", "is_alpha": false }, { "name": "Political Even-handedness", "description": "Anthropic's paired-prompt political bias evaluation with even-handedness, refusal, and hedging metrics.", "category": "safety", "tags": ["politics", "bias", "model-graded", "paired-prompts"], "function_name": "political_evenhandedness", "is_alpha": false }, { "name": "Political Even-handedness: Historical Events", "description": "Political even-handedness evaluation focused on historical events category.", "category": "safety", "tags": ["politics", "bias", "model-graded", "paired-prompts"], "function_name": "political_evenhandedness_historical_events", "is_alpha": false }, { "name": "Political Even-handedness: Policies", "description": "Political even-handedness evaluation focused on policies category.", "category": "safety", "tags": ["politics", "bias", "model-graded", "paired-prompts"], "function_name": "political_evenhandedness_policies", "is_alpha": false }, { "name": "Political Even-handedness: Political Figures", "description": "Political even-handedness evaluation focused on political figures and parties category.", "category": "safety", "tags": ["politics", "bias", "model-graded", "paired-prompts"], "function_name": "political_evenhandedness_political_figures", "is_alpha": false }, { "name": "Political Even-handedness: Scientific Topics", "description": "Political even-handedness evaluation focused on scientific topics category.", "category": "safety", "tags": ["politics", "bias", "model-graded", "paired-prompts"], "function_name": "political_evenhandedness_scientific", "is_alpha": false }, { "name": "Political Even-handedness: Social Issues", "description": "Political even-handedness evaluation focused on social issues category.", "category": "safety", "tags": ["politics", "bias", "model-graded", "paired-prompts"], "function_name": "political_evenhandedness_social_issues", "is_alpha": false }, { "name": "Political Even-handedness: Social and Identity Issues", "description": "Political even-handedness evaluation focused on social and identity issues category.", "category": "safety", "tags": ["politics", "bias", "model-graded", "paired-prompts"], "function_name": "political_evenhandedness_social_identity", "is_alpha": false }, { "name": "Political Even-handedness: US Constitution", "description": "Political even-handedness evaluation focused on US Constitution category.", "category": "safety", "tags": ["politics", "bias", "model-graded", "paired-prompts"], "function_name": "political_evenhandedness_us_constitution", "is_alpha": false }, { "name": "PolygloToxicity: Arabic", "description": "PolygloToxicity toxicity evaluation for Arabic (ar)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_arabic", "is_alpha": false }, { "name": "PolygloToxicity: Chinese", "description": "PolygloToxicity toxicity evaluation for Chinese (zh)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_chinese", "is_alpha": false }, { "name": "PolygloToxicity: Czech", "description": "PolygloToxicity toxicity evaluation for Czech (cs)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_czech", "is_alpha": false }, { "name": "PolygloToxicity: Dutch", "description": "PolygloToxicity toxicity evaluation for Dutch (nl)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_dutch", "is_alpha": false }, { "name": "PolygloToxicity: English", "description": "PolygloToxicity toxicity evaluation for English (en)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_english", "is_alpha": false }, { "name": "PolygloToxicity: French", "description": "PolygloToxicity toxicity evaluation for French (fr)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_french", "is_alpha": false }, { "name": "PolygloToxicity: German", "description": "PolygloToxicity toxicity evaluation for German (de)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_german", "is_alpha": false }, { "name": "PolygloToxicity: Hindi", "description": "PolygloToxicity toxicity evaluation for Hindi (hi)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_hindi", "is_alpha": false }, { "name": "PolygloToxicity: Indonesian", "description": "PolygloToxicity toxicity evaluation for Indonesian (id)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_indonesian", "is_alpha": false }, { "name": "PolygloToxicity: Italian", "description": "PolygloToxicity toxicity evaluation for Italian (it)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_italian", "is_alpha": false }, { "name": "PolygloToxicity: Japanese", "description": "PolygloToxicity toxicity evaluation for Japanese (ja)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_japanese", "is_alpha": false }, { "name": "PolygloToxicity: Korean", "description": "PolygloToxicity toxicity evaluation for Korean (ko)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_korean", "is_alpha": false }, { "name": "PolygloToxicity: Polish", "description": "PolygloToxicity toxicity evaluation for Polish (pl)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_polish", "is_alpha": false }, { "name": "PolygloToxicity: Portuguese", "description": "PolygloToxicity toxicity evaluation for Portuguese (pt)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_portuguese", "is_alpha": false }, { "name": "PolygloToxicity: Russian", "description": "PolygloToxicity toxicity evaluation for Russian (ru)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_russian", "is_alpha": false }, { "name": "PolygloToxicity: Spanish", "description": "PolygloToxicity toxicity evaluation for Spanish (es)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_spanish", "is_alpha": false }, { "name": "PolygloToxicity: Swedish", "description": "PolygloToxicity toxicity evaluation for Swedish (sv)", "category": "polyglotoxicity", "tags": ["toxicity", "safety", "multilingual", "polyglotoxicity"], "function_name": "polyglotoxicity_swedish", "is_alpha": false }, { "name": "PolygloToxicityPrompts (17 Languages)", "description": "Multilingual evaluation of neural toxic degeneration across 17 languages", "category": "core", "tags": ["toxicity", "safety", "multilingual", "generation"], "function_name": "polyglotoxicity", "is_alpha": false }, { "name": "PubMedQA", "description": "Biomedical question answering from PubMed abstracts", "category": "core", "tags": ["multiple-choice", "medical", "biomedical", "research", "literature"], "function_name": "pubmedqa", "is_alpha": false }, { "name": "QA4MRE (All Years)", "description": "Question Answering for Machine Reading Evaluation - CLEF shared tasks 2011-2013", "category": "reading-comprehension", "tags": ["multiple-choice", "reading-comprehension", "clef", "machine-reading"], "function_name": "qa4mre", "is_alpha": false }, { "name": "QA4MRE 2011", "description": "Question Answering for Machine Reading Evaluation (English, 2011)", "category": "reading-comprehension", "tags": ["multiple-choice", "reading-comprehension", "clef", "machine-reading", "qa4mre"], "function_name": "qa4mre_2011", "is_alpha": false }, { "name": "QA4MRE 2012", "description": "Question Answering for Machine Reading Evaluation (English, 2012)", "category": "reading-comprehension", "tags": ["multiple-choice", "reading-comprehension", "clef", "machine-reading", "qa4mre"], "function_name": "qa4mre_2012", "is_alpha": false }, { "name": "QA4MRE 2013", "description": "Question Answering for Machine Reading Evaluation (English, 2013)", "category": "reading-comprehension", "tags": ["multiple-choice", "reading-comprehension", "clef", "machine-reading", "qa4mre"], "function_name": "qa4mre_2013", "is_alpha": false }, { "name": "QASPER (Binary MCQ)", "description": "Question Answering on Scientific Papers - binary yes/no questions on research paper abstracts", "category": "reading-comprehension", "tags": ["multiple-choice", "reading-comprehension", "scientific-papers", "binary-classification"], "function_name": "qasper_ll", "is_alpha": false }, { "name": "RACE", "description": "Reading comprehension from middle and high school English exams (combined)", "category": "reading-comprehension", "tags": ["multiple-choice", "reading-comprehension", "english-exam"], "function_name": "race", "is_alpha": false }, { "name": "RACE-High", "description": "High school level reading comprehension from English exams for Chinese students - passages with multiple questions", "category": "reading-comprehension", "tags": ["multiple-choice", "reading-comprehension", "english-exam", "high-school"], "function_name": "race_high", "is_alpha": false }, { "name": "RACE-Middle", "description": "Middle school level reading comprehension from English exams for Chinese students", "category": "reading-comprehension", "tags": ["multiple-choice", "reading-comprehension", "english-exam", "middle-school"], "function_name": "race_middle", "is_alpha": false }, { "name": "RTE (SuperGLUE)", "description": "Recognizing Textual Entailment from SuperGLUE", "category": "glue", "tags": ["multiple-choice", "superglue", "nli", "reasoning"], "function_name": "rte", "is_alpha": false }, { "name": "RocketScience", "description": "Contrastive spatial reasoning benchmark", "category": "core", "tags": ["multimodal", "spatial-reasoning", "vision-language", "images"], "function_name": "rocketscience", "is_alpha": false }, { "name": "SMT 2024", "description": "Stanford Math Tournament 2024 - competition math problems across multiple categories", "category": "math", "tags": ["smt", "problem-solving", "math", "competition", "2024"], "function_name": "smt", "is_alpha": false }, { "name": "SMT 2024 (Algebra)", "description": "Stanford Math Tournament 2024 - Algebra problems", "category": "math", "tags": ["smt", "algebra", "problem-solving", "math", "2024"], "function_name": "smt_algebra", "is_alpha": false }, { "name": "SMT 2024 (Calculus)", "description": "Stanford Math Tournament 2024 - Calculus problems", "category": "math", "tags": ["smt", "calculus", "problem-solving", "math", "2024"], "function_name": "smt_calculus", "is_alpha": false }, { "name": "SMT 2024 (Discrete)", "description": "Stanford Math Tournament 2024 - Discrete Math problems", "category": "math", "tags": ["smt", "discrete", "problem-solving", "math", "2024"], "function_name": "smt_discrete", "is_alpha": false }, { "name": "SMT 2024 (General)", "description": "Stanford Math Tournament 2024 - General Math problems", "category": "math", "tags": ["smt", "general", "problem-solving", "math", "2024"], "function_name": "smt_general", "is_alpha": false }, { "name": "SMT 2024 (Geometry)", "description": "Stanford Math Tournament 2024 - Geometry problems", "category": "math", "tags": ["smt", "geometry", "problem-solving", "math", "2024"], "function_name": "smt_geometry", "is_alpha": false }, { "name": "SMT 2024 (Guts)", "description": "Stanford Math Tournament 2024 - Guts round problems", "category": "math", "tags": ["smt", "guts", "problem-solving", "math", "2024"], "function_name": "smt_guts", "is_alpha": false }, { "name": "SQuAD v2", "description": "Stanford Question Answering Dataset 2.0 with answerable and unanswerable questions", "category": "core", "tags": ["question-answering", "reading-comprehension", "extractive-qa"], "function_name": "squad_v2", "is_alpha": false }, { "name": "SWAG", "description": "Situations With Adversarial Generations - grounded commonsense inference", "category": "core", "tags": ["multiple-choice", "commonsense-reasoning", "video-captions"], "function_name": "swag", "is_alpha": false }, { "name": "SafeMT-M2S", "description": "SafeMT m2s - Single turn conversion (actor-attack) of the SafeMT dataset", "category": "jailbreak", "tags": ["safemt", "jailbreak", "m2s"], "function_name": "safemt_m2s", "is_alpha": false }, { "name": "SciCode", "description": "Scientific computing and programming challenges", "category": "core", "tags": ["code-generation", "science", "alpha"], "function_name": "scicode", "is_alpha": true }, { "name": "SciQ", "description": "Science exam questions covering Physics, Chemistry, Biology, and other scientific domains", "category": "knowledge-qa", "tags": ["multiple-choice", "science", "physics", "chemistry", "biology"], "function_name": "sciq", "is_alpha": false }, { "name": "SimpleQA", "description": "Measuring short-form factuality in large language models with simple Q&A pairs", "category": "core", "tags": ["factuality", "question-answering", "graded"], "function_name": "simpleqa", "is_alpha": false }, { "name": "SimpleQA Verified", "description": "Rigorously curated benchmark from Google DeepMind measuring short-form factuality with improved data quality, addressing noisy labels, topical biases, and question redundancy", "category": "core", "tags": ["factuality", "question-answering", "graded"], "function_name": "simpleqa_verified", "is_alpha": false }, { "name": "Social IQA", "description": "Social Intelligence Question Answering - tests reasoning about social situations, emotions, and mental states", "category": "ethics-social", "tags": ["multiple-choice", "social-intelligence", "emotional-reasoning", "theory-of-mind"], "function_name": "social_iqa", "is_alpha": false }, { "name": "SuperGLUE (All Tasks)", "description": "SuperGLUE benchmark suite - run any subset by name (boolq, cb, copa, multirc, rte, wic, wsc)", "category": "glue", "tags": ["multiple-choice", "superglue", "nli", "reasoning"], "function_name": "superglue", "is_alpha": false }, { "name": "SuperGPQA", "description": "Scaling LLM Evaluation across 285 Graduate Disciplines - 26,529 multiple-choice questions across science, engineering, medicine, economics, and philosophy", "category": "core", "tags": ["multiple-choice", "knowledge", "graduate-level", "multidisciplinary"], "function_name": "supergpqa", "is_alpha": false }, { "name": "TUMLU", "description": "TUMLU is a comprehensive, multilingual, and natively developed language understanding benchmark specifically designed for Turkic languages.", "category": "community", "tags": ["factuality", "question-answering", "multiple-choice", "reasoning"], "function_name": "tumlu", "is_alpha": false }, { "name": "TauBench Airline", "description": "Tau-bench airline operations benchmark with simulated users and shared tools.", "category": "agents", "tags": ["agents", "tools", "graded"], "function_name": "tau_bench_airline", "is_alpha": true }, { "name": "TauBench Retail", "description": "Tau-bench dual-control customer support simulation in the retail domain.", "category": "agents", "tags": ["agents", "tools", "graded"], "function_name": "tau_bench_retail", "is_alpha": true }, { "name": "TauBench Telecom", "description": "Tau-bench telecom troubleshooting scenarios exercising complex tool flows.", "category": "agents", "tags": ["agents", "tools", "graded"], "function_name": "tau_bench_telecom", "is_alpha": true }, { "name": "Terraform", "description": "Terraform Multiple Choice Questions", "category": "core", "tags": ["code-understanding"], "function_name": "rootly_terraform", "is_alpha": false }, { "name": "ToxiGen", "description": "Toxicity detection benchmark - tests ability to identify toxic and hateful language", "category": "ethics-social", "tags": ["multiple-choice", "toxicity-detection", "hate-speech", "safety"], "function_name": "toxigen", "is_alpha": false }, { "name": "TriviaQA", "description": "Large-scale reading comprehension dataset with trivia questions from Wikipedia and web", "category": "core", "tags": ["question-answering", "trivia", "reading-comprehension"], "function_name": "triviaqa", "is_alpha": false }, { "name": "TruthfulQA", "description": "Tests if models generate truthful answers to questions that humans often answer falsely due to misconceptions", "category": "knowledge-qa", "tags": ["multiple-choice", "truthfulness", "misconceptions", "factuality"], "function_name": "truthfulqa", "is_alpha": false }, { "name": "WSC", "description": "Winograd Schema Challenge - coreference resolution", "category": "glue", "tags": ["multiple-choice", "superglue", "nli", "reasoning", "coreference-resolution"], "function_name": "wsc", "is_alpha": false }, { "name": "WSC273", "description": "Original Winograd Schema Challenge with 273 expert-crafted questions", "category": "core", "tags": ["multiple-choice", "commonsense-reasoning", "pronoun-resolution"], "function_name": "wsc273", "is_alpha": false }, { "name": "WiC", "description": "Word in Context - word sense disambiguation", "category": "glue", "tags": ["multiple-choice", "superglue", "nli", "wsd", "reasoning"], "function_name": "wic", "is_alpha": false }, { "name": "WinoGrande", "description": "Large-scale Winograd Schema Challenge for commonsense pronoun resolution", "category": "core", "tags": ["multiple-choice", "commonsense-reasoning", "pronoun-resolution"], "function_name": "winogrande", "is_alpha": false }, { "name": "XCOPA (11 Languages)", "description": "Cross-lingual Choice of Plausible Alternatives for causal commonsense reasoning", "category": "cross-lingual", "tags": ["multiple-choice", "causal-reasoning", "commonsense", "multilingual"], "function_name": "xcopa", "is_alpha": false }, { "name": "XCOPA: Chinese", "description": "XCOPA causal reasoning for Chinese (zh)", "category": "cross-lingual", "tags": ["multiple-choice", "causal-reasoning", "commonsense", "multilingual", "xcopa"], "function_name": "xcopa_zh", "is_alpha": false }, { "name": "XCOPA: Estonian", "description": "XCOPA causal reasoning for Estonian (et)", "category": "cross-lingual", "tags": ["multiple-choice", "causal-reasoning", "commonsense", "multilingual", "xcopa"], "function_name": "xcopa_et", "is_alpha": false }, { "name": "XCOPA: Haitian Creole", "description": "XCOPA causal reasoning for Haitian Creole (ht)", "category": "cross-lingual", "tags": ["multiple-choice", "causal-reasoning", "commonsense", "multilingual", "xcopa"], "function_name": "xcopa_ht", "is_alpha": false }, { "name": "XCOPA: Indonesian", "description": "XCOPA causal reasoning for Indonesian (id)", "category": "cross-lingual", "tags": ["multiple-choice", "causal-reasoning", "commonsense", "multilingual", "xcopa"], "function_name": "xcopa_id", "is_alpha": false }, { "name": "XCOPA: Italian", "description": "XCOPA causal reasoning for Italian (it)", "category": "cross-lingual", "tags": ["multiple-choice", "causal-reasoning", "commonsense", "multilingual", "xcopa"], "function_name": "xcopa_it", "is_alpha": false }, { "name": "XCOPA: Quechua", "description": "XCOPA causal reasoning for Quechua (qu)", "category": "cross-lingual", "tags": ["multiple-choice", "causal-reasoning", "commonsense", "multilingual", "xcopa"], "function_name": "xcopa_qu", "is_alpha": false }, { "name": "XCOPA: Swahili", "description": "XCOPA causal reasoning for Swahili (sw)", "category": "cross-lingual", "tags": ["multiple-choice", "causal-reasoning", "commonsense", "multilingual", "xcopa"], "function_name": "xcopa_sw", "is_alpha": false }, { "name": "XCOPA: Tamil", "description": "XCOPA causal reasoning for Tamil (ta)", "category": "cross-lingual", "tags": ["multiple-choice", "causal-reasoning", "commonsense", "multilingual", "xcopa"], "function_name": "xcopa_ta", "is_alpha": false }, { "name": "XCOPA: Thai", "description": "XCOPA causal reasoning for Thai (th)", "category": "cross-lingual", "tags": ["multiple-choice", "causal-reasoning", "commonsense", "multilingual", "xcopa"], "function_name": "xcopa_th", "is_alpha": false }, { "name": "XCOPA: Turkish", "description": "XCOPA causal reasoning for Turkish (tr)", "category": "cross-lingual", "tags": ["multiple-choice", "causal-reasoning", "commonsense", "multilingual", "xcopa"], "function_name": "xcopa_tr", "is_alpha": false }, { "name": "XCOPA: Vietnamese", "description": "XCOPA causal reasoning for Vietnamese (vi)", "category": "cross-lingual", "tags": ["multiple-choice", "causal-reasoning", "commonsense", "multilingual", "xcopa"], "function_name": "xcopa_vi", "is_alpha": false }, { "name": "XStoryCloze (11 Languages)", "description": "Cross-lingual story completion for commonsense reasoning", "category": "cross-lingual", "tags": ["multiple-choice", "story-completion", "commonsense", "multilingual"], "function_name": "xstorycloze", "is_alpha": false }, { "name": "XStoryCloze: Arabic", "description": "XStoryCloze story completion for Arabic (ar)", "category": "cross-lingual", "tags": ["multiple-choice", "story-completion", "commonsense", "multilingual", "xstorycloze"], "function_name": "xstorycloze_ar", "is_alpha": false }, { "name": "XStoryCloze: Basque", "description": "XStoryCloze story completion for Basque (eu)", "category": "cross-lingual", "tags": ["multiple-choice", "story-completion", "commonsense", "multilingual", "xstorycloze"], "function_name": "xstorycloze_eu", "is_alpha": false }, { "name": "XStoryCloze: Burmese", "description": "XStoryCloze story completion for Burmese (my)", "category": "cross-lingual", "tags": ["multiple-choice", "story-completion", "commonsense", "multilingual", "xstorycloze"], "function_name": "xstorycloze_my", "is_alpha": false }, { "name": "XStoryCloze: Chinese", "description": "XStoryCloze story completion for Chinese (zh)", "category": "cross-lingual", "tags": ["multiple-choice", "story-completion", "commonsense", "multilingual", "xstorycloze"], "function_name": "xstorycloze_zh", "is_alpha": false }, { "name": "XStoryCloze: English", "description": "XStoryCloze story completion for English (en)", "category": "cross-lingual", "tags": ["multiple-choice", "story-completion", "commonsense", "multilingual", "xstorycloze"], "function_name": "xstorycloze_en", "is_alpha": false }, { "name": "XStoryCloze: Hindi", "description": "XStoryCloze story completion for Hindi (hi)", "category": "cross-lingual", "tags": ["multiple-choice", "story-completion", "commonsense", "multilingual", "xstorycloze"], "function_name": "xstorycloze_hi", "is_alpha": false }, { "name": "XStoryCloze: Indonesian", "description": "XStoryCloze story completion for Indonesian (id)", "category": "cross-lingual", "tags": ["multiple-choice", "story-completion", "commonsense", "multilingual", "xstorycloze"], "function_name": "xstorycloze_id", "is_alpha": false }, { "name": "XStoryCloze: Russian", "description": "XStoryCloze story completion for Russian (ru)", "category": "cross-lingual", "tags": ["multiple-choice", "story-completion", "commonsense", "multilingual", "xstorycloze"], "function_name": "xstorycloze_ru", "is_alpha": false }, { "name": "XStoryCloze: Spanish", "description": "XStoryCloze story completion for Spanish (es)", "category": "cross-lingual", "tags": ["multiple-choice", "story-completion", "commonsense", "multilingual", "xstorycloze"], "function_name": "xstorycloze_es", "is_alpha": false }, { "name": "XStoryCloze: Swahili", "description": "XStoryCloze story completion for Swahili (sw)", "category": "cross-lingual", "tags": ["multiple-choice", "story-completion", "commonsense", "multilingual", "xstorycloze"], "function_name": "xstorycloze_sw", "is_alpha": false }, { "name": "XStoryCloze: Telugu", "description": "XStoryCloze story completion for Telugu (te)", "category": "cross-lingual", "tags": ["multiple-choice", "story-completion", "commonsense", "multilingual", "xstorycloze"], "function_name": "xstorycloze_te", "is_alpha": false }, { "name": "XWinograd (6 Languages)", "description": "Cross-lingual Winograd Schema Challenge for pronoun resolution", "category": "cross-lingual", "tags": ["multiple-choice", "pronoun-resolution", "commonsense", "multilingual"], "function_name": "xwinograd", "is_alpha": false }, { "name": "XWinograd: Chinese", "description": "XWinograd pronoun resolution for Chinese (zh)", "category": "cross-lingual", "tags": ["multiple-choice", "pronoun-resolution", "commonsense", "multilingual", "xwinograd"], "function_name": "xwinograd_zh", "is_alpha": false }, { "name": "XWinograd: English", "description": "XWinograd pronoun resolution for English (en)", "category": "cross-lingual", "tags": ["multiple-choice", "pronoun-resolution", "commonsense", "multilingual", "xwinograd"], "function_name": "xwinograd_en", "is_alpha": false }, { "name": "XWinograd: French", "description": "XWinograd pronoun resolution for French (fr)", "category": "cross-lingual", "tags": ["multiple-choice", "pronoun-resolution", "commonsense", "multilingual", "xwinograd"], "function_name": "xwinograd_fr", "is_alpha": false }, { "name": "XWinograd: Japanese", "description": "XWinograd pronoun resolution for Japanese (jp)", "category": "cross-lingual", "tags": ["multiple-choice", "pronoun-resolution", "commonsense", "multilingual", "xwinograd"], "function_name": "xwinograd_jp", "is_alpha": false }, { "name": "XWinograd: Portuguese", "description": "XWinograd pronoun resolution for Portuguese (pt)", "category": "cross-lingual", "tags": ["multiple-choice", "pronoun-resolution", "commonsense", "multilingual", "xwinograd"], "function_name": "xwinograd_pt", "is_alpha": false }, { "name": "XWinograd: Russian", "description": "XWinograd pronoun resolution for Russian (ru)", "category": "cross-lingual", "tags": ["multiple-choice", "pronoun-resolution", "commonsense", "multilingual", "xwinograd"], "function_name": "xwinograd_ru", "is_alpha": false }, { "name": "sealqa", "description": "Raising the Bar for Reasoning in Search-Augmented Language Models", "category": "core", "tags": ["factuality", "question-answering", "graded"], "function_name": "sealqa", "is_alpha": false }]; ## Featured **57 subjects** · Multiple choice · Knowledge & reasoning Massive Multitask Language Understanding - 57 academic subjects from the cais/mmlu dataset **Graduate-level** · Science · Multiple choice Graduate-level Google-Proof Q\&A in biology, chemistry, and physics **MCP** · Agent · Tool Calling evaluates how effectively language models can navigate and utilize the Model Context Protocol (MCP) ecosystem. **Factuality** · Model-Graded Measuring short-form factuality in large language models with simple Q\&A pairs **Agent** · Code generation Real world code agent programming challenges across 5 languages **Graphs** · Long-context · Reasoning Multi-hop reasoning on graphs - both BFS and parent finding tasks ## Complete Benchmark Catalog export const CatalogTable = () => { const [query, setQuery] = useState(""); const [category, setCategory] = useState("all"); const [tag, setTag] = useState("all"); const [includeAlpha, setIncludeAlpha] = useState(false); const [sortKey, setSortKey] = useState("name"); const [sortDir, setSortDir] = useState("asc"); const [page, setPage] = useState(1); const [pageSize, setPageSize] = useState(20); const data = useMemo(() => { const benchmarks = (benchmarksData || []).map(r => ({ name: String(r?.name ?? "").trim(), description: String(r?.description ?? "").trim(), category: String(r?.category ?? "").trim(), tags: Array.isArray(r?.tags) ? r.tags.map(t => String(t).trim()).filter(Boolean) : [], function_name: String(r?.function_name ?? "").trim(), is_alpha: Boolean(r?.is_alpha), is_group: false, benchmark_count: 0 })); const groups = (evalGroupsData || []).map(g => ({ name: String(g?.name ?? "").trim(), description: String(g?.description ?? "").trim(), category: String(g?.category ?? "").trim(), tags: Array.isArray(g?.tags) ? g.tags.map(t => String(t).trim()).filter(Boolean) : [], function_name: String(g?.id ?? "").trim(), is_alpha: false, is_group: true, benchmark_count: Number(g?.benchmark_count ?? 0) })); return [...benchmarks, ...groups]; }, []); const categories = useMemo(() => { const set = new Set(); data.forEach(r => set.add(r.category)); return Array.from(set).sort((a, b) => a.localeCompare(b)); }, [data]); const tags = useMemo(() => { const set = new Set(); data.forEach(r => r.tags.forEach(t => set.add(t))); return Array.from(set).sort((a, b) => a.localeCompare(b)); }, [data]); const filtered = useMemo(() => { const q = query.trim().toLowerCase(); const byQuery = r => !q ? true : r.name.toLowerCase().includes(q) || r.description.toLowerCase().includes(q) || r.tags.some(t => t.toLowerCase().includes(q)) || r.function_name.toLowerCase().includes(q); const byCategory = r => category === "all" ? true : r.category === category; const byTag = r => tag === "all" ? true : r.tags.includes(tag); const byAlpha = r => includeAlpha ? true : !r.is_alpha; return data.filter(r => byQuery(r) && byCategory(r) && byTag(r) && byAlpha(r)); }, [data, query, category, tag, includeAlpha]); const sorted = useMemo(() => { const s = [...filtered]; s.sort((a, b) => { const av = a[sortKey]; const bv = b[sortKey]; const cmp = String(av).localeCompare(String(bv)); return sortDir === "asc" ? cmp : -cmp; }); return s; }, [filtered, sortKey, sortDir]); const total = sorted.length; const totalPages = Math.max(1, Math.ceil(total / pageSize)); const currentPage = Math.min(page, totalPages); const startIdx = (currentPage - 1) * pageSize; const pageRows = sorted.slice(startIdx, startIdx + pageSize); useEffect(() => { setPage(1); }, [query, category, tag, includeAlpha, pageSize]); const toggleSort = key => { if (key === sortKey) setSortDir(d => d === "asc" ? "desc" : "asc"); else { setSortKey(key); setSortDir("asc"); } }; const icon = active => ; return
{filtered.length} results
{pageRows.length === 0 ? : pageRows.map(r => )}
Description Tags Command
No results match your filters.
{r.name}{r.is_alpha && alpha}{r.is_group && group}
{r.is_group && r.benchmark_count > 0 &&
{r.benchmark_count} benchmarks
}		 {r.description} {r.category}
{r.tags.map(t => {t})}
 bench eval {r.function_name}
Page {currentPage} of {totalPages}
; };
## Grader Information Some benchmarks use a grader model to score the model's performance. This requires an additional API key for the grader model. To run these benchmarks, you'll need to export your `OPENAI_API_key`: ```bash theme={null} export OPENAI_API_KEY=your_openai_key ``` The following benchmarks use a grader model: | Benchmark | Default Grader Model | | :------------------------- | :------------------------------- | | `simpleqa` | `openai/gpt-4.1-2025-04-14` | | `hle` | `openai/o3-mini-2025-01-31` | | `hle_text` | `openai/o3-mini-2025-01-31` | | `browsecomp` | `openai/gpt-4.1-2025-04-14` | | `healthbench` | `openai/gpt-4.1-2025-04-14` | | `math` | `openai/gpt-4-turbo-preview` | | `math_500` | `openai/gpt-4-turbo-preview` | | `detailbench` | `gpt-5-mini-2025-08-07` | | `livemcpbench` | `openai/gpt-4.1-mini-2025-04-14` | | `otis_mock_aime` | `openai/gpt-4.1-mini-2025-04-14` | | `political_evenhandedness` | `openai/gpt-4.1-2025-04-14` |