{"id":"https://openalex.org/W4396630607","doi":"https://doi.org/10.48550/arxiv.2405.00622","title":"Causal Evaluation of Language Models","display_name":"Causal Evaluation of Language Models","publication_year":2024,"publication_date":"2024-05-01","ids":{"openalex":"https://openalex.org/W4396630607","doi":"https://doi.org/10.48550/arxiv.2405.00622"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2405.00622","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.00622","pdf_url":"https://arxiv.org/pdf/2405.00622","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2405.00622","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104335504","display_name":"Sirui Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Sirui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036493016","display_name":"Bo Peng","orcid":"https://orcid.org/0000-0002-7626-8455"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Bo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045639716","display_name":"Meiqi Chen","orcid":"https://orcid.org/0009-0004-9010-7836"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Meiqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100387935","display_name":"Ruiqi Wang","orcid":"https://orcid.org/0000-0001-8763-8573"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Ruiqi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102008835","display_name":"Mengying Xu","orcid":"https://orcid.org/0000-0003-1862-6709"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Mengying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113197478","display_name":"Xingyu Zeng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Xingyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041519196","display_name":"Rui Zhao","orcid":"https://orcid.org/0009-0000-9709-0994"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Rui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035948567","display_name":"Shengjie Zhao","orcid":"https://orcid.org/0000-0002-6109-2522"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Shengjie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100748135","display_name":"Yu Qiao","orcid":"https://orcid.org/0000-0002-1889-2567"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiao, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5050188515","display_name":"Chaochao Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Chaochao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.5289000272750854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.5289000272750854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4503999948501587,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.43549999594688416,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4945347011089325},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.38453900814056396},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3774205446243286},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.11643171310424805}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4945347011089325},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.38453900814056396},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3774205446243286},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.11643171310424805}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2405.00622","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.00622","pdf_url":"https://arxiv.org/pdf/2405.00622","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2405.00622","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2405.00622","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2405.00622","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2405.00622","pdf_url":"https://arxiv.org/pdf/2405.00622","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W4395014643","https://openalex.org/W4391913857","https://openalex.org/W2350741829"],"abstract_inverted_index":{"Causal":[0,39],"reasoning":[1,61],"is":[2,52,262],"viewed":[3],"as":[4,236],"crucial":[5],"for":[6,31,57,148,209,240],"achieving":[7],"human-level":[8],"machine":[9],"intelligence.":[10],"Recent":[11],"advances":[12],"in":[13],"language":[14,64,161,211],"models":[15,162],"have":[16],"expanded":[17],"the":[18,47,53,59,69,92,99,107,127,185,241],"horizons":[19],"of":[20,41,49,63,78,138,158,167,184],"artificial":[21],"intelligence":[22],"across":[23,188,201],"various":[24,189],"domains,":[25],"sparking":[26],"inquiries":[27],"into":[28],"their":[29],"potential":[30],"causal":[32,60,81,139,169,247],"reasoning.":[33],"In":[34],"this":[35],"work,":[36],"we":[37,67,125,153,180,195,215],"introduce":[38],"evaluation":[40,115,157,186],"Language":[42],"Models":[43],"(CaLM),":[44],"which,":[45],"to":[46,85,90,97,105,134,227,255],"best":[48],"our":[50],"knowledge,":[51],"first":[54],"comprehensive":[55],"benchmark":[56,239],"evaluating":[58],"capabilities":[62],"models.":[65],"First,":[66],"propose":[68],"CaLM":[70,128,235],"framework,":[71],"which":[72],"establishes":[73],"a":[74,113,164,217,221],"foundational":[75],"taxonomy":[76,111],"consisting":[77],"four":[79],"modules:":[80],"target":[82],"(i.e.,":[83,88,95,103],"what":[84],"evaluate),":[86],"adaptation":[87],"how":[89,96,104],"obtain":[91],"results),":[93,100],"metric":[94],"measure":[98],"and":[101,122,143,175,225,230,252],"error":[102,177,253],"analyze":[106],"bad":[108],"results).":[109],"This":[110],"defines":[112],"broad":[114],"design":[116],"space":[117],"while":[118],"systematically":[119,243],"selecting":[120],"criteria":[121],"priorities.":[123],"Second,":[124],"compose":[126],"dataset,":[129],"comprising":[130],"126,334":[131],"data":[132],"samples,":[133],"provide":[135],"curated":[136],"sets":[137],"targets,":[140,170,248],"adaptations,":[141,172,249],"metrics,":[142,174,251],"errors,":[144],"offering":[145],"extensive":[146,156],"coverage":[147],"diverse":[149],"research":[150,258],"pursuits.":[151],"Third,":[152],"conduct":[154],"an":[155,237],"28":[159],"leading":[160],"on":[163],"core":[165],"set":[166],"92":[168],"9":[171,202],"7":[173],"12":[176],"types.":[178],"Fourth,":[179],"perform":[181],"detailed":[182],"analyses":[183],"results":[187],"dimensions":[190,203],"(e.g.,":[191,204],"adaptation,":[192],"scale).":[193],"Fifth,":[194],"present":[196],"50":[197],"high-level":[198],"empirical":[199],"findings":[200],"model),":[205],"providing":[206],"valuable":[207],"guidance":[208],"future":[210],"model":[212],"development.":[213],"Finally,":[214],"develop":[216],"multifaceted":[218],"platform,":[219],"including":[220],"website,":[222],"leaderboards,":[223],"datasets,":[224],"toolkits,":[226],"support":[228],"scalable":[229],"adaptable":[231],"assessments.":[232],"We":[233],"envision":[234],"ever-evolving":[238],"community,":[242],"updated":[244],"with":[245],"new":[246],"models,":[250],"types":[254],"reflect":[256],"ongoing":[257],"advancements.":[259],"Project":[260],"website":[261],"at":[263],"https://opencausalab.github.io/CaLM.":[264]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2}],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-10T00:00:00"}