{"id":"https://openalex.org/W4411638714","doi":"https://doi.org/10.18653/v1/2024.eacl-short.22","title":"Predict the Next Word: &lt;Humans exhibit uncertainty in this task and language models _____&gt;","display_name":"Predict the Next Word: &lt;Humans exhibit uncertainty in this task and language models _____&gt;","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4411638714","doi":"https://doi.org/10.18653/v1/2024.eacl-short.22"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2024.eacl-short.22","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.eacl-short.22","pdf_url":"https://aclanthology.org/2024.eacl-short.22.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2024.eacl-short.22.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092574996","display_name":"Evgenia Ilia","orcid":null},"institutions":[{"id":"https://openalex.org/I55106644","display_name":"Amsterdam University of Applied Sciences","ror":"https://ror.org/00y2z2s03","country_code":"NL","type":"education","lineage":["https://openalex.org/I55106644"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Evgenia Ilia","raw_affiliation_strings":["University of Amsterdam University of Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam University of Amsterdam","institution_ids":["https://openalex.org/I55106644","https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074142241","display_name":"Wilker Aziz","orcid":"https://orcid.org/0000-0002-2093-3866"},"institutions":[{"id":"https://openalex.org/I55106644","display_name":"Amsterdam University of Applied Sciences","ror":"https://ror.org/00y2z2s03","country_code":"NL","type":"education","lineage":["https://openalex.org/I55106644"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Wilker Aziz","raw_affiliation_strings":["University of Amsterdam University of Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam University of Amsterdam","institution_ids":["https://openalex.org/I55106644","https://openalex.org/I887064364"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2825,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.67475176,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"234","last_page":"255"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.4634000062942505,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.4634000062942505,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7120804786682129},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6157512068748474},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.5996332764625549},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5753876566886902},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49624496698379517},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4461395740509033},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4297778010368347},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.30560484528541565},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08041062951087952}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7120804786682129},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6157512068748474},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.5996332764625549},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5753876566886902},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49624496698379517},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4461395740509033},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4297778010368347},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.30560484528541565},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08041062951087952},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.18653/v1/2024.eacl-short.22","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.eacl-short.22","pdf_url":"https://aclanthology.org/2024.eacl-short.22.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)","raw_type":"proceedings-article"},{"id":"pmh:oai:dare.uva.nl:publications/f10096be-ad2e-48bf-becd-099a6e1c1945","is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/predict-the-next-word(f10096be-ad2e-48bf-becd-099a6e1c1945).html","pdf_url":"https://pure.uva.nl/ws/files/241804867/2024.eacl-short.22v2.pdf","source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Ilia, E & Aziz, W 2024, Predict the Next Word: < Humans exhibit uncertainty in this task and language models _____ > in Y Graham & M Purver (eds), The 18th Conference of the European Chapter of the Association for Computational Linguistics : proceedings of the conference : EACL 2024 : March 17-22, 2024. vol. 2, Kerrville, TX, pp. 234-255, 18th Conference of the European Chapter of the Association for Computational Linguistics, EACL 2024, St. Julian\ufffds, Malta, 17/03/24. https://doi.org/10.18653/v1/2024.eacl-short.22","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.18653/v1/2024.eacl-short.22","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.eacl-short.22","pdf_url":"https://aclanthology.org/2024.eacl-short.22.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 2: Short Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320334322","display_name":"HORIZON EUROPE Framework Programme","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4411638714.pdf","grobid_xml":"https://content.openalex.org/works/W4411638714.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W1569283511","https://openalex.org/W2169518243","https://openalex.org/W4236193183","https://openalex.org/W2053866214","https://openalex.org/W2607505004","https://openalex.org/W2231795205","https://openalex.org/W2523632547","https://openalex.org/W2296205523","https://openalex.org/W2944691285","https://openalex.org/W3098003361"],"abstract_inverted_index":{"Language":[0],"models":[1,5],"(LMs)":[2],"are":[3],"statistical":[4,30],"trained":[6],"to":[7,10,17,34,93,135,151,163],"assign":[8],"probability":[9],"humangenerated":[11],"text.As":[12],"such,":[13,168],"it":[14,41,175],"is":[15,32,54],"reasonable":[16],"question":[18],"whether":[19],"they":[20,146],"approximate":[21],"linguistic":[22],"variability":[23,95],"exhibited":[24],"by":[25],"humans":[26,97],"well.This":[27],"form":[28,118],"of":[29,77,81,102,119,125,158],"assessment":[31],"difficult":[33],"perform":[35],"at":[36],"the":[37,56,82,90,107,123,156,170],"passage":[38],"level,":[39,58],"for":[40],"requires":[42],"acceptability":[43],"judgments":[44],"(i.e.,":[45],"human":[46,136,152],"evaluation)":[47],"or":[48],"a":[49,74,100,117],"robust":[50],"automated":[51],"proxy":[52],"(which":[53],"non-trivial).At":[55],"word":[57,109],"however,":[59],"given":[60],"some":[61],"context,":[62],"samples":[63],"from":[64],"an":[65],"LM":[66],"can":[67,112],"be":[68,113],"assessed":[69],"via":[70],"exact":[71],"matching":[72],"against":[73,172],"prerecorded":[75],"dataset":[76],"alternative":[78],"single-word":[79],"continuations":[80],"available":[83],"context.We":[84],"exploit":[85],"this":[86,177],"fact":[87],"and":[88,141,143,166],"evaluate":[89],"LM's":[91],"ability":[92],"reproduce":[94],"that":[96,145],"(in":[98],"particular,":[99],"population":[101],"English":[103],"speakers)":[104],"exhibit":[105,147],"in":[106,122,176],"'next":[108],"prediction'":[110],"task.This":[111],"seen":[114],"as":[115,167],"assessing":[116],"calibration,":[120],"which,":[121],"context":[124],"text":[126],"classification,":[127],"Baan":[128],"et":[129],"al.":[130],"(":[131],"2022)":[132],"termed":[133],"calibration":[134,150,160],"uncertainty.We":[137,153],"assess":[138],"GPT2,":[139],"BLOOM":[140],"ChatGPT":[142],"find":[144],"fairly":[148],"low":[149],"also":[154],"verify":[155],"failure":[157],"expected":[159],"error":[161],"(ECE)":[162],"reflect":[164],"this,":[165],"advise":[169],"community":[171],"relying":[173],"on":[174],"setting.":[178]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-02T09:51:11.867554","created_date":"2025-10-10T00:00:00"}