{"id":"https://openalex.org/W3194676777","doi":"https://doi.org/10.1145/3474381","title":"WinoGrande","display_name":"WinoGrande","publication_year":2021,"publication_date":"2021-08-24","ids":{"openalex":"https://openalex.org/W3194676777","doi":"https://doi.org/10.1145/3474381","mag":"3194676777"},"language":"en","primary_location":{"id":"doi:10.1145/3474381","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3474381","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3474381","source":{"id":"https://openalex.org/S103482838","display_name":"Communications of the ACM","issn_l":"0001-0782","issn":["0001-0782","1557-7317"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Communications of the ACM","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3474381","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101067919","display_name":"Keisuke Sakaguchi","orcid":"https://orcid.org/0000-0002-3809-1732"},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Keisuke Sakaguchi","raw_affiliation_strings":["Allen Institute for AI, Seattle, WA"],"affiliations":[{"raw_affiliation_string":"Allen Institute for AI, Seattle, WA","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024879161","display_name":"Ronan Le Bras","orcid":"https://orcid.org/0000-0003-2439-6938"},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ronan Le Bras","raw_affiliation_strings":["Allen Institute for AI, Seattle, WA"],"affiliations":[{"raw_affiliation_string":"Allen Institute for AI, Seattle, WA","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044250030","display_name":"Chandra Bhagavatula","orcid":"https://orcid.org/0000-0001-6264-0378"},"institutions":[{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chandra Bhagavatula","raw_affiliation_strings":["Allen Institute for AI, Seattle, WA"],"affiliations":[{"raw_affiliation_string":"Allen Institute for AI, Seattle, WA","institution_ids":["https://openalex.org/I4210140341"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102992157","display_name":"Yejin Choi","orcid":"https://orcid.org/0000-0003-3032-5378"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yejin Choi","raw_affiliation_strings":["University of Washington & Allen Institute for AI, Seattle, WA"],"affiliations":[{"raw_affiliation_string":"University of Washington & Allen Institute for AI, Seattle, WA","institution_ids":["https://openalex.org/I4210140341","https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5101067919"],"corresponding_institution_ids":["https://openalex.org/I4210140341"],"apc_list":null,"apc_paid":null,"fwci":16.5157,"has_fulltext":true,"cited_by_count":266,"citation_normalized_percentile":{"value":0.9931743,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"64","issue":"9","first_page":"99","last_page":"106"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/commonsense-reasoning","display_name":"Commonsense reasoning","score":0.8167423009872437},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7883296012878418},{"id":"https://openalex.org/keywords/spurious-relationship","display_name":"Spurious relationship","score":0.7737498879432678},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.6459641456604004},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6083164215087891},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5487788319587708},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5337508916854858},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5069534778594971},{"id":"https://openalex.org/keywords/schema","display_name":"Schema (genetic algorithms)","score":0.45826205611228943},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.43485429883003235},{"id":"https://openalex.org/keywords/commonsense-knowledge","display_name":"Commonsense knowledge","score":0.4170539975166321},{"id":"https://openalex.org/keywords/knowledge-extraction","display_name":"Knowledge extraction","score":0.0889616310596466}],"concepts":[{"id":"https://openalex.org/C193221554","wikidata":"https://www.wikidata.org/wiki/Q5153664","display_name":"Commonsense reasoning","level":2,"score":0.8167423009872437},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7883296012878418},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.7737498879432678},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.6459641456604004},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6083164215087891},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5487788319587708},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5337508916854858},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5069534778594971},{"id":"https://openalex.org/C52146309","wikidata":"https://www.wikidata.org/wiki/Q7431116","display_name":"Schema (genetic algorithms)","level":2,"score":0.45826205611228943},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.43485429883003235},{"id":"https://openalex.org/C30542707","wikidata":"https://www.wikidata.org/wiki/Q1603203","display_name":"Commonsense knowledge","level":3,"score":0.4170539975166321},{"id":"https://openalex.org/C120567893","wikidata":"https://www.wikidata.org/wiki/Q1582085","display_name":"Knowledge extraction","level":2,"score":0.0889616310596466},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3474381","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3474381","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3474381","source":{"id":"https://openalex.org/S103482838","display_name":"Communications of the ACM","issn_l":"0001-0782","issn":["0001-0782","1557-7317"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Communications of the ACM","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3474381","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3474381","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3474381","source":{"id":"https://openalex.org/S103482838","display_name":"Communications of the ACM","issn_l":"0001-0782","issn":["0001-0782","1557-7317"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Communications of the ACM","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"},{"id":"https://openalex.org/F4320338406","display_name":"Naval Information Warfare Center Pacific","ror":"https://ror.org/01gs1cg95"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3194676777.pdf","grobid_xml":"https://content.openalex.org/works/W3194676777.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W95183648","https://openalex.org/W359091458","https://openalex.org/W1599016936","https://openalex.org/W1752492850","https://openalex.org/W2001771035","https://openalex.org/W2050482109","https://openalex.org/W2073302931","https://openalex.org/W2145482038","https://openalex.org/W2145755360","https://openalex.org/W2251035762","https://openalex.org/W2296266385","https://openalex.org/W2395141554","https://openalex.org/W2402954262","https://openalex.org/W2406611863","https://openalex.org/W2579903943","https://openalex.org/W2786351140","https://openalex.org/W2805206884","https://openalex.org/W2806710540","https://openalex.org/W2888161220","https://openalex.org/W2899501643","https://openalex.org/W2945290257","https://openalex.org/W2952984539","https://openalex.org/W2962736243","https://openalex.org/W2962843521","https://openalex.org/W2963096121","https://openalex.org/W2963120843","https://openalex.org/W2963159690","https://openalex.org/W2963167649","https://openalex.org/W2963457723","https://openalex.org/W2964532710","https://openalex.org/W2969282158","https://openalex.org/W2990704537","https://openalex.org/W3035032873","https://openalex.org/W3099655892","https://openalex.org/W6601893370"],"related_works":["https://openalex.org/W3035583586","https://openalex.org/W4320165839","https://openalex.org/W2151799802","https://openalex.org/W4386607580","https://openalex.org/W4385488510","https://openalex.org/W2196562041","https://openalex.org/W2073302931","https://openalex.org/W4378501473","https://openalex.org/W3082691151","https://openalex.org/W4287633646"],"abstract_inverted_index":{"Commonsense":[0],"reasoning":[1],"remains":[2],"a":[3,37,95,138,214],"major":[4],"challenge":[5],"in":[6,73,182,239],"AI,":[7],"and":[8,113],"yet,":[9],"recent":[10,22],"progresses":[11],"on":[12,31,51,70,162,173,191,197,226,247],"benchmarks":[13,194,229],"may":[14],"seem":[15],"to":[16,42,78,108,147,165,233],"suggest":[17],"otherwise.":[18],"In":[19],"particular,":[20],"the":[21,32,74,82,103,111,114,117,123,170,174,183,202,207,220,223,231],"neural":[23],"language":[24],"models":[25,47,60,156],"have":[26,61],"reported":[27],"above":[28],"90%":[29],"accuracy":[30,160],"Winograd":[33],"Schema":[34],"Challenge":[35],"(WSC),":[36],"commonsense":[38,65],"benchmark":[39],"originally":[40],"designed":[41],"be":[43],"unsolvable":[44],"for":[45,216],"statistical":[46],"that":[48,76,142,154,169],"rely":[49,69],"simply":[50],"word":[52,145],"associations.":[53,150],"This":[54],"raises":[55],"an":[56,79],"important":[57],"question---whether":[58],"these":[59,228],"truly":[62],"acquired":[63],"robust":[64],"capabilities":[66,84],"or":[67],"they":[68,205],"spurious":[71,180,235],"biases":[72,181,236],"dataset":[75,97,124],"lead":[77],"overestimation":[80],"of":[81,85,98,116,122,127,209],"true":[83],"machine":[86],"commonsense.":[87],"To":[88],"investigate":[89],"this":[90],"question,":[91],"we":[92,186],"introduce":[93],"WinoGrande,":[94],"large-scale":[96,129],"44k":[99],"problems,":[100],"inspired":[101],"by":[102,132,179],"original":[104,175],"WSC,":[105],"but":[106],"adjusted":[107],"improve":[109],"both":[110],"scale":[112],"hardness":[115],"dataset.":[118,184],"The":[119],"key":[120],"steps":[121],"construction":[125],"consist":[126],"(1)":[128],"crowdsourcing,":[130],"followed":[131],"(2)":[133],"systematic":[134],"bias":[135,249],"reduction":[136],"using":[137],"novel":[139],"AFLITE":[140],"algorithm":[141],"generalizes":[143],"human-detectable":[144],"associations":[146],"machine-detectable":[148],"embedding":[149],"Our":[151],"experiments":[152],"demonstrate":[153,206],"state-of-the-art":[155,189],"achieve":[157],"considerably":[158],"lower":[159],"(59.4%-79.1%)":[161],"WINOGRANDE":[163,210],"compared":[164],"humans":[166],"(94%),":[167],"confirming":[168],"high":[171,224],"performance":[172,225],"WSC":[176],"was":[177],"inflated":[178],"Furthermore,":[185],"report":[187],"new":[188],"results":[190],"five":[192],"related":[193],"with":[195],"emphasis":[196],"their":[198],"dual":[199],"implications.":[200],"On":[201,219],"one":[203],"hand,":[204,222],"effectiveness":[208],"when":[211],"used":[212],"as":[213],"resource":[215],"transfer":[217],"learning.":[218],"other":[221],"all":[227,240],"suggests":[230],"extent":[232],"which":[234,243],"are":[237],"prevalent":[238],"such":[241],"datasets,":[242],"motivates":[244],"further":[245],"research":[246],"algorithmic":[248],"reduction.":[250]},"counts_by_year":[{"year":2026,"cited_by_count":19},{"year":2025,"cited_by_count":129},{"year":2024,"cited_by_count":74},{"year":2023,"cited_by_count":32},{"year":2022,"cited_by_count":10},{"year":2021,"cited_by_count":2}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2021-08-30T00:00:00"}