{"id":"https://openalex.org/W4392340595","doi":"https://doi.org/10.48550/arxiv.2402.18225","title":"CogBench: a large language model walks into a psychology lab","display_name":"CogBench: a large language model walks into a psychology lab","publication_year":2024,"publication_date":"2024-02-28","ids":{"openalex":"https://openalex.org/W4392340595","doi":"https://doi.org/10.48550/arxiv.2402.18225"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2402.18225","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.18225","pdf_url":"https://arxiv.org/pdf/2402.18225","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2402.18225","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089420217","display_name":"Julian Coda-Forno","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Coda-Forno, Julian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001129714","display_name":"Marcel Binz","orcid":"https://orcid.org/0000-0001-8872-8386","observed_orcids":["https://orcid.org/0000-0001-8872-8386"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Binz, Marcel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054833182","display_name":"Jane X. Wang","orcid":"https://orcid.org/0000-0003-1339-5040","observed_orcids":["https://orcid.org/0000-0003-1339-5040"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jane X.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5101768980","display_name":"Eric Schulz","orcid":"https://orcid.org/0000-0003-3088-0371","observed_orcids":["https://orcid.org/0000-0003-3088-0371"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schulz, Eric","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":53,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7461000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7461000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.14830000698566437,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.01679999940097332,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9660000205039978},{"id":"https://openalex.org/keywords/llm-benchmarking","display_name":"LLM benchmarking","score":0.7639999985694885},{"id":"https://openalex.org/keywords/behavioral-metrics","display_name":"behavioral metrics","score":0.6959999799728394},{"id":"https://openalex.org/keywords/cognitive-psychology-experiments","display_name":"cognitive psychology experiments","score":0.9409999847412109},{"id":"https://openalex.org/keywords/model-size","display_name":"model size","score":0.6330000162124634},{"id":"https://openalex.org/keywords/reinforcement-learning-from-human-feedback","display_name":"reinforcement learning from human feedback","score":0.9940000176429749},{"id":"https://openalex.org/keywords/open-source-vs-closed-source-models","display_name":"open-source vs closed-source models","score":0.921999990940094},{"id":"https://openalex.org/keywords/prompt-engineering","display_name":"prompt engineering","score":0.8889999985694885},{"id":"https://openalex.org/keywords/chain-of-thought-prompting","display_name":"chain-of-thought prompting","score":0.9929999709129333},{"id":"https://openalex.org/keywords/probabilistic-reasoning","display_name":"probabilistic reasoning","score":0.9100000262260437},{"id":"https://openalex.org/keywords/model-based-reasoning","display_name":"model-based reasoning","score":0.777999997138977},{"id":"https://openalex.org/keywords/multilevel-modeling","display_name":"multilevel modeling","score":0.7710000276565552}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.47677353024482727},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.35883694887161255}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2402.18225","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.18225","pdf_url":"https://arxiv.org/pdf/2402.18225","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:push-zb.helmholtz-munich.de:71772","doi":null,"is_oa":false,"landing_page_url":"https://push-zb.helmholtz-munich.de/frontdoor.php?source_opus=71772","pdf_url":null,"source":{"id":"https://openalex.org/S7407055352","display_name":"PuSH - Publication Server of Helmholtz Zentrum M\u00fcnchen","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"In:. 2024. 9076-9108 ( ; 235)","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"doi:10.48550/arxiv.2402.18225","doi":"https://doi.org/10.48550/arxiv.2402.18225","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.18225","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2402.18225","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.18225","pdf_url":"https://arxiv.org/pdf/2402.18225","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4392340595.pdf","grobid_xml":"https://content.openalex.org/works/W4392340595.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W2146196757","https://openalex.org/W3001279689","https://openalex.org/W3128912454","https://openalex.org/W3195577433","https://openalex.org/W4226024653","https://openalex.org/W4283026156","https://openalex.org/W4321277158","https://openalex.org/W4327810158","https://openalex.org/W4360836968","https://openalex.org/W4366850553","https://openalex.org/W4377121468","https://openalex.org/W4383313862","https://openalex.org/W4384918448","https://openalex.org/W4389395901","https://openalex.org/W4389611357","https://openalex.org/W4400681340","https://openalex.org/W4403122979","https://openalex.org/W7071831927","https://openalex.org/W7133219754","https://openalex.org/W7133230388","https://openalex.org/W7133239970"],"related_works":["https://openalex.org/W4400065236","https://openalex.org/W7213392355","https://openalex.org/W7134831769","https://openalex.org/W4414758425","https://openalex.org/W4415343286","https://openalex.org/W7125508116","https://openalex.org/W6979302937","https://openalex.org/W7148698518","https://openalex.org/W7208737449","https://openalex.org/W7162813444"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,123,129],"(LLMs)":[3],"have":[4],"significantly":[5],"advanced":[6],"the":[7,26,85,97,144],"field":[8],"of":[9,91,100,146],"artificial":[10],"intelligence.":[11],"Yet,":[12],"evaluating":[13],"them":[14],"comprehensively":[15],"remains":[16],"challenging.":[17],"We":[18,62,74,149],"argue":[19],"that":[20,41,121,131,151],"this":[21,76],"is":[22],"partly":[23],"due":[24],"to":[25,65],"predominant":[27],"focus":[28],"on":[29,133],"performance":[30,112],"metrics":[31,45],"in":[32,110],"most":[33],"benchmarks.":[34],"This":[35,52],"paper":[36],"introduces":[37],"CogBench,":[38],"a":[39,56,69],"benchmark":[40],"includes":[42],"ten":[43],"behavioral":[44],"derived":[46],"from":[47,106],"seven":[48],"cognitive":[49],"psychology":[50],"experiments.":[51],"novel":[53],"approach":[54],"offers":[55],"toolkit":[57],"for":[58,84],"phenotyping":[59],"LLMs'":[60,139],"behavior.":[61,117,140],"apply":[63],"CogBench":[64],"35":[66],"LLMs,":[67],"yielding":[68],"rich":[70],"and":[71,103,113,130],"diverse":[72],"dataset.":[73],"analyze":[75],"data":[77],"using":[78],"statistical":[79],"multilevel":[80],"modeling":[81],"techniques,":[82],"accounting":[83],"nested":[86],"dependencies":[87],"among":[88],"fine-tuned":[89],"versions":[90],"specific":[92],"LLMs.":[93],"Our":[94],"study":[95],"highlights":[96],"crucial":[98],"role":[99],"model":[101],"size":[102],"reinforcement":[104],"learning":[105],"human":[107,116],"feedback":[108],"(RLHF)":[109],"improving":[111],"aligning":[114],"with":[115],"Interestingly,":[118],"we":[119,142],"find":[120],"open-source":[122],"are":[124],"less":[125],"risk-prone":[126],"than":[127],"proprietary":[128],"fine-tuning":[132],"code":[134],"does":[135],"not":[136],"necessarily":[137],"enhance":[138],"Finally,":[141],"explore":[143],"effects":[145],"prompt-engineering":[147],"techniques.":[148],"discover":[150],"chain-of-thought":[152],"prompting":[153,159],"improves":[154],"probabilistic":[155],"reasoning,":[156],"while":[157],"take-a-step-back":[158],"fosters":[160],"model-based":[161],"behaviors.":[162]},"counts_by_year":[{"year":2026,"cited_by_count":14},{"year":2025,"cited_by_count":23},{"year":2024,"cited_by_count":16}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-03-05T00:00:00"}