{"id":"https://openalex.org/W4389519908","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.504","title":"POSQA: Probe the World Models of LLMs with Size Comparisons","display_name":"POSQA: Probe the World Models of LLMs with Size Comparisons","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389519908","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.504"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.504","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.504","is_oa":true,"landing_page_url":"http://dx.doi.org/10.18653/v1/2023.findings-emnlp.504","pdf_url":"https://aclanthology.org/2023.findings-emnlp.504.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.findings-emnlp.504.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Chang Shu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]},{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU","GB"],"is_corresponding":false,"raw_author_name":"Chang Shu","raw_affiliation_strings":["University of Cambridge","Google DeepMind","Monash University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge","institution_ids":["https://openalex.org/I241749"]},{"raw_affiliation_string":"Google DeepMind","institution_ids":["https://openalex.org/I4210090411"]},{"raw_affiliation_string":"Monash University","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058074512","display_name":"Jiuzhou Han","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]},{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU","GB"],"is_corresponding":false,"raw_author_name":"Jiuzhou Han","raw_affiliation_strings":["Monash University","Google DeepMind","University of Cambridge"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Monash University","institution_ids":["https://openalex.org/I56590836"]},{"raw_affiliation_string":"Google DeepMind","institution_ids":["https://openalex.org/I4210090411"]},{"raw_affiliation_string":"University of Cambridge","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026154387","display_name":"Fangyu Liu","orcid":"https://orcid.org/0000-0001-7038-3623","observed_orcids":["https://orcid.org/0000-0001-7038-3623"]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]},{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU","GB"],"is_corresponding":false,"raw_author_name":"Fangyu Liu","raw_affiliation_strings":["University of Cambridge","Google DeepMind","Monash University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge","institution_ids":["https://openalex.org/I241749"]},{"raw_affiliation_string":"Google DeepMind","institution_ids":["https://openalex.org/I4210090411"]},{"raw_affiliation_string":"Monash University","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086032589","display_name":"Ehsan Shareghi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]},{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU","GB"],"is_corresponding":false,"raw_author_name":"Ehsan Shareghi","raw_affiliation_strings":["Monash University","Google DeepMind","University of Cambridge"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Monash University","institution_ids":["https://openalex.org/I56590836"]},{"raw_affiliation_string":"Google DeepMind","institution_ids":["https://openalex.org/I4210090411"]},{"raw_affiliation_string":"University of Cambridge","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5073413742","display_name":"Nigel Collier","orcid":"https://orcid.org/0000-0002-7230-4164","observed_orcids":["https://orcid.org/0000-0002-7230-4164"]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]},{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU","GB"],"is_corresponding":false,"raw_author_name":"Nigel Collier","raw_affiliation_strings":["University of Cambridge","Google DeepMind","Monash University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Cambridge","institution_ids":["https://openalex.org/I241749"]},{"raw_affiliation_string":"Google DeepMind","institution_ids":["https://openalex.org/I4210090411"]},{"raw_affiliation_string":"Monash University","institution_ids":["https://openalex.org/I56590836"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.3314,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.53253921,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"7518","last_page":"7531"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.725600004196167,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.725600004196167,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.23109999299049377,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.026799999177455902,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9589999914169312},{"id":"https://openalex.org/keywords/world-models","display_name":"world models","score":0.6499999761581421},{"id":"https://openalex.org/keywords/zero-shot-reasoning","display_name":"zero-shot reasoning","score":0.5699999928474426},{"id":"https://openalex.org/keywords/prompt-engineering","display_name":"prompt engineering","score":0.47699999809265137}],"concepts":[{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.7471112608909607},{"id":"https://openalex.org/C100609095","wikidata":"https://www.wikidata.org/wiki/Q1335050","display_name":"Embodied cognition","level":2,"score":0.7355766892433167},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.47450634837150574},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.4640144109725952},{"id":"https://openalex.org/C2777365542","wikidata":"https://www.wikidata.org/wiki/Q83090","display_name":"Avatar","level":2,"score":0.4563366770744324},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.4255138337612152},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.42539331316947937},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.35337066650390625},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2632768750190735},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.21610796451568604},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.18653/v1/2023.findings-emnlp.504","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.504","is_oa":true,"landing_page_url":"http://dx.doi.org/10.18653/v1/2023.findings-emnlp.504","pdf_url":"https://aclanthology.org/2023.findings-emnlp.504.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2310.13394","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.13394","pdf_url":"https://arxiv.org/pdf/2310.13394","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:monash.edu:openaire/17e8c107-800b-446a-90ba-ba1412b05fa6","doi":null,"is_oa":true,"landing_page_url":"https://research.monash.edu/en/publications/17e8c107-800b-446a-90ba-ba1412b05fa6","pdf_url":null,"source":{"id":"https://openalex.org/S4306402625","display_name":"Monash University Research Portal (Monash University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I56590836","host_organization_name":"Monash University","host_organization_lineage":["https://openalex.org/I56590836"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Shu, C, Han, J, Liu, F, Shareghi, E & Collier, N 2023, POSQA : Probe the World Models of LLMs with Size Comparisons. in N Tomeh, A Fujita, A Sun, B Wang, R Tong & R Cotterell (eds), The 2023 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics (ACL), Stroudsburg PA USA, pp. 7518\u20137531, Empirical Methods in Natural Language Processing 2023, Singapore, 6/12/23. < https://aclanthology.org/2023.findings-emnlp.504 >","raw_type":"contributionToPeriodical"},{"id":"doi:10.48550/arxiv.2310.13394","doi":"https://doi.org/10.48550/arxiv.2310.13394","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.13394","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.findings-emnlp.504","doi":"https://doi.org/10.18653/v1/2023.findings-emnlp.504","is_oa":true,"landing_page_url":"http://dx.doi.org/10.18653/v1/2023.findings-emnlp.504","pdf_url":"https://aclanthology.org/2023.findings-emnlp.504.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2023","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G761817724","display_name":"Knowledge Grounded Dialogue Agents","funder_award_id":"2752931","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320314731","display_name":"UK Research and Innovation","ror":"https://ror.org/001aqnf71"},{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389519908.pdf","grobid_xml":"https://content.openalex.org/works/W4389519908.grobid-xml"},"referenced_works_count":40,"referenced_works":["https://openalex.org/W136732505","https://openalex.org/W1967933442","https://openalex.org/W2011323112","https://openalex.org/W2150375089","https://openalex.org/W2156279557","https://openalex.org/W2165087100","https://openalex.org/W2481532481","https://openalex.org/W2507087756","https://openalex.org/W2626804490","https://openalex.org/W2981852735","https://openalex.org/W2998617917","https://openalex.org/W3020712669","https://openalex.org/W3030163527","https://openalex.org/W3082115681","https://openalex.org/W3121904249","https://openalex.org/W3175526286","https://openalex.org/W3198002980","https://openalex.org/W3205068155","https://openalex.org/W3207553988","https://openalex.org/W4226278401","https://openalex.org/W4305028650","https://openalex.org/W4307079201","https://openalex.org/W4311408938","https://openalex.org/W4317553041","https://openalex.org/W4317672148","https://openalex.org/W4317840265","https://openalex.org/W4318919287","https://openalex.org/W4319049323","https://openalex.org/W4319165647","https://openalex.org/W4320167623","https://openalex.org/W4323572061","https://openalex.org/W4385302156","https://openalex.org/W4389520747","https://openalex.org/W4392935324","https://openalex.org/W4394828156","https://openalex.org/W7094054136","https://openalex.org/W7106552862","https://openalex.org/W7133224126","https://openalex.org/W7133231263","https://openalex.org/W7133231369"],"related_works":["https://openalex.org/W4404343443","https://openalex.org/W4391128605","https://openalex.org/W7014557847","https://openalex.org/W4412491123","https://openalex.org/W4385571962","https://openalex.org/W4389518393","https://openalex.org/W7155576675","https://openalex.org/W4412945046","https://openalex.org/W4389520366","https://openalex.org/W7128915081"],"abstract_inverted_index":{"Embodied":[0],"language":[1,5],"comprehension":[2,84,119],"emphasises":[3],"that":[4,90,143,146],"understanding":[6,145],"is":[7,46],"not":[8],"only":[9],"a":[10,61],"matter":[11],"of":[12,31,81,85,132,138,163],"mental":[13],"processing":[14],"in":[15,41],"the":[16,23,28,74,78,82,86,92,99,130,160],"brain,":[17],"but":[18],"also":[19],"involves":[20],"interactions":[21],"with":[22,67,106,170],"physical":[24],"and":[25,36,76,110,128,135,157],"social":[26],"environment.With":[27],"explosive":[29],"growth":[30],"Large":[32],"Language":[33],"Models":[34],"(LLMs)":[35],"their":[37,52,104,117],"already":[38],"ubiquitous":[39],"presence":[40],"our":[42],"daily":[43],"lives,":[44],"it":[45,167],"becoming":[47],"increasingly":[48],"necessary":[49],"to":[50,72,155],"verify":[51],"real-world":[53,118,144],"understanding.Inspired":[54],"by":[55,159],"cognitive":[56],"theories,":[57],"we":[58,114],"propose":[59],"POSQA:":[60],"Physical":[62],"Object":[63],"Size":[64],"Question-Answering":[65],"dataset":[66],"simple":[68],"size":[69],"comparison":[70],"questions":[71],"examine":[73],"extremity":[75],"analyse":[77,129],"potential":[79],"mechanisms":[80],"embodied":[83],"latest":[87],"LLM.We":[88],"show":[89,142],"even":[91],"largest":[93],"LLMs":[94,147],"today":[95],"perform":[96],"poorly":[97],"under":[98],"zero-shot":[100],"setting.We":[101],"then":[102],"push":[103],"limits":[105],"advanced":[107],"prompting":[108],"techniques":[109],"external":[111],"knowledge":[112],"augmentation.Furthermore,":[113],"investigate":[115],"whether":[116],"primarily":[120],"derives":[121],"from":[122,149],"contextual":[123],"information":[124],"or":[125],"internal":[126],"weights":[127],"impact":[131],"prompt":[133],"formats":[134],"report":[136],"bias":[137],"different":[139],"objects.Our":[140],"results":[141],"shaped":[148],"textual":[150],"data":[151],"can":[152],"be":[153],"vulnerable":[154],"deception":[156],"confusion":[158],"surface":[161],"form":[162],"prompts,":[164],"which":[165],"makes":[166],"less":[168],"aligned":[169],"human":[171],"behaviours.":[172]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2023-12-11T00:00:00"}