{"id":"https://openalex.org/W4392181802","doi":"https://doi.org/10.48550/arxiv.2402.14865","title":"Dynamic Evaluation of Large Language Models by Meta Probing Agents","display_name":"Dynamic Evaluation of Large Language Models by Meta Probing Agents","publication_year":2024,"publication_date":"2024-02-21","ids":{"openalex":"https://openalex.org/W4392181802","doi":"https://doi.org/10.48550/arxiv.2402.14865"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2402.14865","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.14865","pdf_url":"https://arxiv.org/pdf/2402.14865","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2402.14865","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103056110","display_name":"Kaijie Zhu","orcid":"https://orcid.org/0009-0002-6220-1476","observed_orcids":["https://orcid.org/0009-0002-6220-1476"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhu, Kaijie","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114179770","display_name":"Jindong Wang","orcid":"https://orcid.org/0009-0007-9762-589X","observed_orcids":["https://orcid.org/0009-0007-9762-589X"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang, Jindong","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104331798","display_name":"Qinlin Zhao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhao, Qinlin","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093973962","display_name":"Ruochen Xu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xu, Ruochen","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5125430482","display_name":"Xing Xie","orcid":"https://orcid.org/0009-0009-3257-3077","observed_orcids":["https://orcid.org/0009-0009-3257-3077"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xie, Xing","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I126520041"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8984000086784363,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8984000086784363,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.05860000103712082,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.00800000037997961,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.9190000295639038},{"id":"https://openalex.org/keywords/psychometrics","display_name":"psychometrics","score":0.6420000195503235},{"id":"https://openalex.org/keywords/data-contamination","display_name":"contaminated data","score":0.5879999995231628},{"id":"https://openalex.org/keywords/language-comprehension","display_name":"language comprehension","score":0.6949999928474426},{"id":"https://openalex.org/keywords/problem-solving","display_name":"problem solving","score":0.9959999918937683},{"id":"https://openalex.org/keywords/domain-knowledge","display_name":"domain knowledge","score":0.9980000257492065},{"id":"https://openalex.org/keywords/matthew-effect","display_name":"Matthew effect","score":0.828000009059906},{"id":"https://openalex.org/keywords/model-size","display_name":"model size","score":0.7490000128746033},{"id":"https://openalex.org/keywords/data-augmentation","display_name":"data augmentation","score":0.8090000152587891}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.49923205375671387}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2402.14865","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.14865","pdf_url":"https://arxiv.org/pdf/2402.14865","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2402.14865","doi":"https://doi.org/10.48550/arxiv.2402.14865","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.14865","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2402.14865","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.14865","pdf_url":"https://arxiv.org/pdf/2402.14865","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4392181802.pdf","grobid_xml":"https://content.openalex.org/works/W4392181802.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4387294159","https://openalex.org/W4391421129","https://openalex.org/W4399062248","https://openalex.org/W7133236894","https://openalex.org/W7134047175","https://openalex.org/W7080044819","https://openalex.org/W4392011697","https://openalex.org/W4410392463","https://openalex.org/W7077935843","https://openalex.org/W7160618021"],"abstract_inverted_index":{"Evaluation":[0],"of":[1,17,58,86,181],"large":[2],"language":[3,121],"models":[4,177],"(LLMs)":[5],"has":[6],"raised":[7],"great":[8],"concerns":[9],"in":[10],"the":[11,15,46,83,92,97,160,164,182],"community":[12],"due":[13],"to":[14,36,78,102,194],"issue":[16],"data":[18,191],"contamination.":[19],"Existing":[20],"work":[21],"designed":[22],"evaluation":[23,41,73,107],"protocols":[24],"using":[25,142],"well-defined":[26],"algorithms":[27],"for":[28,154],"specific":[29],"tasks,":[30],"which":[31,89],"cannot":[32,51],"be":[33,187],"easily":[34],"extended":[35],"diverse":[37],"scenarios.":[38],"Moreover,":[39],"current":[40],"benchmarks":[42],"can":[43,185],"only":[44],"provide":[45],"overall":[47],"benchmark":[48],"results":[49],"and":[50,55,99,125,144,167],"support":[52],"a":[53,70,110,190],"fine-grained":[54],"multifaceted":[56,136,157],"analysis":[57,158],"LLMs'":[59],"abilities.":[60,183],"In":[61],"this":[62],"paper,":[63],"we":[64],"propose":[65],"meta":[66],"probing":[67,98],"agents":[68,101],"(MPA),":[69],"general":[71],"dynamic":[72],"protocol":[74],"inspired":[75],"by":[76],"psychometrics":[77],"evaluate":[79],"LLMs.":[80,196],"MPA":[81,95,143,184],"is":[82,198],"key":[84],"component":[85],"DyVal":[87],"2,":[88],"naturally":[90],"extends":[91],"previous":[93],"DyVal~\\citep{zhu2023dyval}.":[94],"designs":[96],"judging":[100],"automatically":[103],"transform":[104],"an":[105,168],"original":[106],"problem":[108,123],"into":[109],"new":[111],"one":[112],"following":[113],"psychometric":[114],"theory":[115],"on":[116,172],"three":[117],"basic":[118,129,165],"cognitive":[119],"abilities:":[120],"understanding,":[122],"solving,":[124],"domain":[126],"knowledge.":[127],"These":[128],"abilities":[130,166],"are":[131],"also":[132,186],"dynamically":[133],"configurable,":[134],"allowing":[135],"analysis.":[137],"We":[138],"conducted":[139],"extensive":[140],"evaluations":[141],"found":[145],"that":[146],"most":[147],"LLMs":[148],"achieve":[149],"poorer":[150],"performance,":[151],"indicating":[152],"room":[153],"improvement.":[155],"Our":[156],"demonstrated":[159],"strong":[161],"correlation":[162],"between":[163],"implicit":[169],"Matthew":[170],"effect":[171],"model":[173],"size,":[174],"i.e.,":[175],"larger":[176],"possess":[178],"stronger":[179],"correlations":[180],"used":[188],"as":[189],"augmentation":[192],"approach":[193],"enhance":[195],"Code":[197],"available":[199],"at:":[200],"https://github.com/microsoft/promptbench.":[201]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-09-30T09:27:27.532063","created_date":"2024-02-27T00:00:00"}