{"id":"https://openalex.org/W7164541770","doi":"https://doi.org/10.48550/arxiv.2606.12730","title":"Rethinking Psychometric Evaluation of LLMs: When and Why Self-Reports Predict Behavior","display_name":"Rethinking Psychometric Evaluation of LLMs: When and Why Self-Reports Predict Behavior","publication_year":2026,"publication_date":"2026-06-10","ids":{"openalex":"https://openalex.org/W7164541770","doi":"https://doi.org/10.48550/arxiv.2606.12730"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.12730","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12730","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.12730","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136514225","display_name":"Rafal Kocielnik","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kocielnik, Rafal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138495716","display_name":"Pengrui Han","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Pengrui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114152107","display_name":"Peiyang Song","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Peiyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071949195","display_name":"Myrl G. Marmarelis","orcid":"https://orcid.org/0000-0001-8742-7848"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Marmarelis, Myrl G.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007263620","display_name":"Ramit Debnath","orcid":"https://orcid.org/0000-0003-0727-5683"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Debnath, Ramit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128290705","display_name":"Dean Mobbs","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mobbs, Dean","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138545620","display_name":"Anima Anandkumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Anandkumar, Anima","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138506559","display_name":"R. Michael Alvarez","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alvarez, R. Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.15410000085830688,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10525","display_name":"Human-Automation Interaction and Safety","score":0.15410000085830688,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12607","display_name":"Personal Information Management and User Behavior","score":0.0957999974489212,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11519","display_name":"Digital Mental Health Interventions","score":0.08380000293254852,"subfield":{"id":"https://openalex.org/subfields/3202","display_name":"Applied Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/personality","display_name":"Personality","score":0.5507000088691711},{"id":"https://openalex.org/keywords/coherence","display_name":"Coherence (philosophical gambling strategy)","score":0.5457000136375427},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.46970000863075256},{"id":"https://openalex.org/keywords/big-five-personality-traits","display_name":"Big Five personality traits","score":0.46549999713897705},{"id":"https://openalex.org/keywords/theory-of-planned-behavior","display_name":"Theory of planned behavior","score":0.35370001196861267},{"id":"https://openalex.org/keywords/isolation","display_name":"Isolation (microbiology)","score":0.3352999985218048},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.3165000081062317}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.7078999876976013},{"id":"https://openalex.org/C187288502","wikidata":"https://www.wikidata.org/wiki/Q641118","display_name":"Personality","level":2,"score":0.5507000088691711},{"id":"https://openalex.org/C2781181686","wikidata":"https://www.wikidata.org/wiki/Q4226068","display_name":"Coherence (philosophical gambling strategy)","level":2,"score":0.5457000136375427},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.5414999723434448},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.5177000164985657},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.46970000863075256},{"id":"https://openalex.org/C2865642","wikidata":"https://www.wikidata.org/wiki/Q378132","display_name":"Big Five personality traits","level":3,"score":0.46549999713897705},{"id":"https://openalex.org/C89785696","wikidata":"https://www.wikidata.org/wiki/Q2290753","display_name":"Theory of planned behavior","level":3,"score":0.35370001196861267},{"id":"https://openalex.org/C2775941552","wikidata":"https://www.wikidata.org/wiki/Q25212305","display_name":"Isolation (microbiology)","level":2,"score":0.3352999985218048},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3165000081062317},{"id":"https://openalex.org/C20685875","wikidata":"https://www.wikidata.org/wiki/Q7239678","display_name":"Predictive validity","level":2,"score":0.3147999942302704},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.30320000648498535},{"id":"https://openalex.org/C174010058","wikidata":"https://www.wikidata.org/wiki/Q7170643","display_name":"Personality Assessment Inventory","level":3,"score":0.28780001401901245},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.26840001344680786},{"id":"https://openalex.org/C2778355321","wikidata":"https://www.wikidata.org/wiki/Q17079427","display_name":"Identity (music)","level":2,"score":0.26100000739097595},{"id":"https://openalex.org/C121244757","wikidata":"https://www.wikidata.org/wiki/Q1783487","display_name":"Convergent validity","level":4,"score":0.2587999999523163}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.12730","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12730","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.12730","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12730","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Anticipating":[0],"LLM":[1],"behavioral":[2,108],"tendencies":[3],"from":[4],"low-cost":[5],"psychometric":[6],"probes":[7],"is":[8,129,173],"critical":[9],"for":[10,155,214],"safe":[11],"deployment,":[12],"but":[13,29,128,190],"only":[14,154],"if":[15],"self-reports":[16,185],"(SR)":[17],"reliably":[18],"predict":[19,38],"behavior.":[20,217],"Recent":[21],"work":[22],"documented":[23],"substantial":[24],"SR-behavior":[25,125],"dissociation":[26],"in":[27,43],"LLMs,":[28,113],"relied":[30],"on":[31],"broad":[32,101],"personality":[33,202],"traits":[34],"(Big":[35],"5)":[36],"that":[37,124,200],"specific":[39,92],"behaviors":[40,156],"weakly,":[41],"even":[42,226],"humans.":[44],"Furthermore,":[45],"the":[46,65,80,136,159,211],"isolation":[47],"of":[48,82,138],"conversational":[49],"sessions":[50],"combined":[51],"with":[52,79,179],"weak":[53],"context":[54,118],"matching":[55],"left":[56],"open":[57],"whether":[58,64],"LLMs":[59],"truly":[60],"lack":[61],"coherence":[62,71,126,152],"or":[63],"conditions":[66],"needed":[67],"to":[68,90],"detect":[69],"such":[70,162,204],"were":[72],"not":[73,192,209],"met.":[74],"We":[75,103,122],"contrast":[76],"Big":[77,144,206],"5":[78,145,207],"Theory":[81,137],"Planned":[83,139],"Behavior":[84,140],"(TPB),":[85],"which":[86],"measures":[87],"intention":[88],"targeted":[89],"a":[91,133],"behavior":[93,97,172,194],"and":[94,110,119,169,220,225,233],"predicts":[95],"human":[96],"substantially":[98],"better":[99],"than":[100],"traits.":[102],"run":[104],"experiments":[105],"across":[106,188,231],"four":[107],"tasks":[109,232],"11":[111],"frontier":[112],"while":[114],"also":[115],"varying":[116],"session":[117],"identity":[120],"induction.":[121],"find":[123],"exists":[127],"selective.":[130],"1)":[131],"Within":[132],"shared":[134],"conversation,":[135],"reaches":[141],"human-level":[142],"coherence;":[143],"does":[146,191],"not.":[147],"2)":[148],"Across":[149],"separate":[150],"conversations,":[151,189],"survives":[153],"anchored":[157],"outside":[158],"immediate":[160],"prompt,":[161],"as":[163,178,205],"implicit":[164],"bias":[165],"shaped":[166],"by":[167,176],"training,":[168],"collapses":[170],"when":[171],"strongly":[174],"primed":[175],"context,":[177],"sycophancy.":[180],"3)":[181],"Persona":[182],"prompting":[183],"makes":[184],"more":[186],"consistent":[187],"bring":[193],"into":[195],"alignment.":[196],"These":[197],"findings":[198],"suggest":[199],"coarse":[201],"frameworks,":[203],"may":[208],"be":[210,229],"best":[212],"tools":[213],"testing":[215],"deployment":[216],"More":[218],"task-":[219],"behavior-specific":[221],"instruments":[222],"are":[223],"needed,":[224],"these":[227],"must":[228],"evaluated":[230],"contexts.":[234]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-13T00:00:00"}