{"id":"https://openalex.org/W4387964152","doi":"https://doi.org/10.48550/arxiv.2310.16755","title":"HI-TOM: A Benchmark for Evaluating Higher-Order Theory of Mind Reasoning in Large Language Models","display_name":"HI-TOM: A Benchmark for Evaluating Higher-Order Theory of Mind Reasoning in Large Language Models","publication_year":2023,"publication_date":"2023-10-25","ids":{"openalex":"https://openalex.org/W4387964152","doi":"https://doi.org/10.48550/arxiv.2310.16755"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2310.16755","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.16755","pdf_url":"https://arxiv.org/pdf/2310.16755","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2310.16755","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100839368","display_name":"Yinghui He","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"He, Yinghui","raw_affiliation_strings":["University of Michigan Westlake University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan Westlake University","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101208758","display_name":"Yufan Wu","orcid":"https://orcid.org/0009-0009-9002-2118","observed_orcids":["https://orcid.org/0009-0009-9002-2118"]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wu, Yufan","raw_affiliation_strings":["University of Michigan Westlake University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan Westlake University","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100571209","display_name":"Yilin Jia","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jia, Yilin","raw_affiliation_strings":["University of Michigan Westlake University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan Westlake University","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082450455","display_name":"Rada F. Mihalcea","orcid":"https://orcid.org/0000-0002-0767-6703","observed_orcids":["https://orcid.org/0000-0002-0767-6703"]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mihalcea, Rada","raw_affiliation_strings":["University of Michigan Westlake University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan Westlake University","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100777438","display_name":"Yulong Chen","orcid":"https://orcid.org/0000-0002-4960-5241","observed_orcids":["https://orcid.org/0000-0002-4960-5241","https://orcid.org/0009-0002-1623-6627"]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chen, Yulong","raw_affiliation_strings":["University of Michigan Westlake University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan Westlake University","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007774447","display_name":"Naihao Deng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Deng, Naihao","raw_affiliation_strings":["University of Michigan Westlake University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Michigan Westlake University","institution_ids":["https://openalex.org/I27837315"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I27837315"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":37,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10656","display_name":"Child and Animal Learning Development","score":0.9136000275611877,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10656","display_name":"Child and Animal Learning Development","score":0.9136000275611877,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.01810000091791153,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11010","display_name":"Logic, Reasoning, and Knowledge","score":0.016699999570846558,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recursive-reasoning","display_name":"recursive reasoning","score":0.5809999704360962},{"id":"https://openalex.org/keywords/mental-state-reasoning","display_name":"mental state reasoning","score":0.6650000214576721},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.753000020980835},{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.8009999990463257},{"id":"https://openalex.org/keywords/llm-limitations","display_name":"LLM limitations","score":0.5529999732971191},{"id":"https://openalex.org/keywords/failure-case-analysis","display_name":"failure case analysis","score":0.6510000228881836}],"concepts":[{"id":"https://openalex.org/C2779560602","wikidata":"https://www.wikidata.org/wiki/Q639219","display_name":"Theory of mind","level":3,"score":0.8089295625686646},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.6696828603744507},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6248335838317871},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.4804222583770752},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.46105897426605225},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.42344266176223755},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.38001811504364014},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.12902188301086426},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2310.16755","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.16755","pdf_url":"https://arxiv.org/pdf/2310.16755","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.16755","doi":"https://doi.org/10.48550/arxiv.2310.16755","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.16755","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2310.16755","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.16755","pdf_url":"https://arxiv.org/pdf/2310.16755","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320337253","display_name":"Instituto de Ciencias del Mar y Limnolog\u00eda, Universidad Nacional Aut\u00f3noma de M\u00e9xico","ror":"https://ror.org/040wdye98"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387964152.pdf","grobid_xml":"https://content.openalex.org/works/W4387964152.grobid-xml"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W2889107415","https://openalex.org/W4321277158","https://openalex.org/W4327526719","https://openalex.org/W4360836968","https://openalex.org/W4385572854","https://openalex.org/W4387835442","https://openalex.org/W4389519585","https://openalex.org/W7131797834","https://openalex.org/W7133224126","https://openalex.org/W7133233472"],"related_works":["https://openalex.org/W4392181976","https://openalex.org/W6979354344","https://openalex.org/W4416644609","https://openalex.org/W4407571040","https://openalex.org/W7125508116","https://openalex.org/W4406512427","https://openalex.org/W7117966739","https://openalex.org/W4389519164","https://openalex.org/W7131673088","https://openalex.org/W4403344747"],"abstract_inverted_index":{"Theory":[0,59],"of":[1,24,60,84,92,96,105,111],"Mind":[2,61],"(ToM)":[3],"is":[4],"the":[5,22,82,103,109],"ability":[6],"to":[7],"reason":[8],"about":[9],"one's":[10],"own":[11],"and":[12,28,39,98],"others'":[13,51],"mental":[14],"states.":[15],"ToM":[16,79],"plays":[17],"a":[18,56,73,89],"critical":[19],"role":[20],"in":[21,75],"development":[23],"intelligence,":[25],"language":[26],"understanding,":[27],"cognitive":[29],"processes.":[30],"While":[31],"previous":[32],"work":[33],"has":[34],"primarily":[35],"focused":[36],"on":[37,50,77,102,108],"first":[38],"second-order":[40],"ToM,":[41,45],"we":[42],"explore":[43],"higher-order":[44,78],"which":[46],"involves":[47],"recursive":[48],"reasoning":[49],"beliefs.":[52],"We":[53,87],"introduce":[54],"HI-TOM,":[55],"Higher":[57],"Order":[58],"benchmark.":[62],"Our":[63],"experimental":[64],"evaluation":[65],"using":[66],"various":[67],"Large":[68],"Language":[69],"Models":[70],"(LLMs)":[71],"indicates":[72],"decline":[74],"performance":[76],"tasks,":[80],"demonstrating":[81],"limitations":[83],"current":[85],"LLMs.":[86],"conduct":[88],"thorough":[90],"analysis":[91],"different":[93],"failure":[94],"cases":[95],"LLMs,":[97],"share":[99],"our":[100,106],"thoughts":[101],"implications":[104],"findings":[107],"future":[110],"NLP.":[112]},"counts_by_year":[{"year":2026,"cited_by_count":12},{"year":2025,"cited_by_count":21},{"year":2024,"cited_by_count":4}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2023-10-27T00:00:00"}