{"id":"https://openalex.org/W2996908057","doi":"https://doi.org/10.1609/aaai.v34i05.6399","title":"WinoGrande: An Adversarial Winograd Schema Challenge at Scale","display_name":"WinoGrande: An Adversarial Winograd Schema Challenge at Scale","publication_year":2020,"publication_date":"2020-04-03","ids":{"openalex":"https://openalex.org/W2996908057","doi":"https://doi.org/10.1609/aaai.v34i05.6399","mag":"2996908057"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v34i05.6399","doi":"https://doi.org/10.1609/aaai.v34i05.6399","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v34i05.6399","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1609/aaai.v34i05.6399","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101067919","display_name":"Keisuke Sakaguchi","orcid":"https://orcid.org/0000-0002-3809-1732","observed_orcids":["https://orcid.org/0000-0002-3809-1732"]},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Keisuke Sakaguchi","raw_affiliation_strings":["Allen Institute for AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for AI","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024879161","display_name":"Ronan Joseph Le Bras","orcid":"https://orcid.org/0000-0003-2439-6938","observed_orcids":["https://orcid.org/0000-0003-2439-6938","https://orcid.org/0000-0001-6784-1312"]},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ronan Le Bras","raw_affiliation_strings":["Allen Institute for AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for AI","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044250030","display_name":"Chandra Bhagavatula","orcid":"https://orcid.org/0000-0001-6264-0378","observed_orcids":["https://orcid.org/0000-0001-6264-0378"]},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chandra Bhagavatula","raw_affiliation_strings":["Allen Institute for AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for AI","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102992157","display_name":"Yejin Choi","orcid":"https://orcid.org/0000-0003-3032-5378","observed_orcids":["https://orcid.org/0000-0003-3032-5378"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yejin Choi","raw_affiliation_strings":["University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":28.4728,"has_fulltext":true,"cited_by_count":1067,"citation_normalized_percentile":{"value":0.99786658,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"34","issue":"05","first_page":"8732","last_page":"8740"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7304999828338623,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7304999828338623,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.19840000569820404,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.031700000166893005,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/winograd-schema-challenge","display_name":"Winograd Schema Challenge","score":0.9769999980926514},{"id":"https://openalex.org/keywords/commonsense-reasoning","display_name":"commonsense reasoning","score":0.871999979019165},{"id":"https://openalex.org/keywords/pronoun-resolution","display_name":"pronoun resolution","score":0.9670000076293945},{"id":"https://openalex.org/keywords/dataset-construction","display_name":"dataset construction","score":0.5830000042915344},{"id":"https://openalex.org/keywords/bias-mitigation","display_name":"bias mitigation","score":0.6499999761581421},{"id":"https://openalex.org/keywords/word-association","display_name":"word association","score":0.7099999785423279},{"id":"https://openalex.org/keywords/spurious-correlations","display_name":"spurious correlations","score":0.4909999966621399},{"id":"https://openalex.org/keywords/transfer-learning","display_name":"transfer learning","score":0.6579999923706055},{"id":"https://openalex.org/keywords/copa","display_name":"COPA","score":0.531000018119812},{"id":"https://openalex.org/keywords/dpr","display_name":"DPR","score":0.6679999828338623},{"id":"https://openalex.org/keywords/large-scale-datasets","display_name":"large-scale datasets","score":0.35499998927116394}],"concepts":[{"id":"https://openalex.org/C193221554","wikidata":"https://www.wikidata.org/wiki/Q5153664","display_name":"Commonsense reasoning","level":2,"score":0.8300048112869263},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7342709898948669},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.6222952604293823},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5859549641609192},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5195327401161194},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.49818897247314453},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.49598798155784607},{"id":"https://openalex.org/C52146309","wikidata":"https://www.wikidata.org/wiki/Q7431116","display_name":"Schema (genetic algorithms)","level":2,"score":0.4912802577018738},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4767756760120392},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.45888644456863403},{"id":"https://openalex.org/C2777462759","wikidata":"https://www.wikidata.org/wiki/Q18395344","display_name":"Word embedding","level":3,"score":0.43105271458625793},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1609/aaai.v34i05.6399","doi":"https://doi.org/10.1609/aaai.v34i05.6399","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v34i05.6399","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1907.10641","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1907.10641","pdf_url":"https://arxiv.org/pdf/1907.10641","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:ojs.aaai.org:article/6399","doi":null,"is_oa":true,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/6399","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/6399/6255","source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"},{"id":"doi:10.48550/arxiv.1907.10641","doi":"https://doi.org/10.48550/arxiv.1907.10641","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1907.10641","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1609/aaai.v34i05.6399","doi":"https://doi.org/10.1609/aaai.v34i05.6399","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v34i05.6399","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1037014486","display_name":null,"funder_award_id":"N66001","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G5310676827","display_name":"RI: Small: A Data-Driven Framework to Sketch-to-Text Generation","funder_award_id":"1524371","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5385673730","display_name":null,"funder_award_id":"W911NF-15-1-","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G5551313627","display_name":"RI: Small: ConnotationNet: Modeling Non-Literal Meaning in Context","funder_award_id":"1714566","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7357862116","display_name":null,"funder_award_id":"N66001-19-2-4031","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G7452299184","display_name":null,"funder_award_id":"W911NF","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G8215844777","display_name":null,"funder_award_id":"N66001-19","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G8257659198","display_name":null,"funder_award_id":"W911NF-15","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"},{"id":"https://openalex.org/F4320338406","display_name":"Naval Information Warfare Center Pacific","ror":"https://ror.org/01gs1cg95"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2996908057.pdf","grobid_xml":"https://content.openalex.org/works/W2996908057.grobid-xml"},"referenced_works_count":46,"referenced_works":["https://openalex.org/W85083507","https://openalex.org/W95183648","https://openalex.org/W359091458","https://openalex.org/W1599016936","https://openalex.org/W1752492850","https://openalex.org/W2050482109","https://openalex.org/W2103651919","https://openalex.org/W2129657639","https://openalex.org/W2145482038","https://openalex.org/W2145755360","https://openalex.org/W2155247542","https://openalex.org/W2169455193","https://openalex.org/W2250668331","https://openalex.org/W2251035762","https://openalex.org/W2251158365","https://openalex.org/W2296266385","https://openalex.org/W2395141554","https://openalex.org/W2402954262","https://openalex.org/W2406611863","https://openalex.org/W2526182867","https://openalex.org/W2579903943","https://openalex.org/W2786351140","https://openalex.org/W2790415926","https://openalex.org/W2796868841","https://openalex.org/W2798280648","https://openalex.org/W2798358706","https://openalex.org/W2798617936","https://openalex.org/W2805206884","https://openalex.org/W2806710540","https://openalex.org/W2886424491","https://openalex.org/W2888161220","https://openalex.org/W2896457183","https://openalex.org/W2898695519","https://openalex.org/W2899501643","https://openalex.org/W2945290257","https://openalex.org/W2952984539","https://openalex.org/W2955503152","https://openalex.org/W2963341956","https://openalex.org/W2963424663","https://openalex.org/W2963661177","https://openalex.org/W2965373594","https://openalex.org/W2969282158","https://openalex.org/W2970946372","https://openalex.org/W2989033444","https://openalex.org/W2990704537","https://openalex.org/W3188177599"],"related_works":["https://openalex.org/W3194676777","https://openalex.org/W3022402349","https://openalex.org/W2183859253","https://openalex.org/W3099859597","https://openalex.org/W3023908169","https://openalex.org/W2951034424","https://openalex.org/W2945290257","https://openalex.org/W3154295049","https://openalex.org/W2395141554","https://openalex.org/W3211543788"],"abstract_inverted_index":{"The":[0,124,159],"Winograd":[1],"Schema":[2],"Challenge":[3],"(WSC)":[4],"(Levesque,":[5],"Davis,":[6],"and":[7,118,212,273],"Morgenstern":[8],"2011),":[9],"a":[10,16,99,134,146,233,244],"benchmark":[11],"for":[12,29,235],"commonsense":[13,69,258],"reasoning,":[14],"is":[15],"set":[17],"of":[18,55,86,90,102,121,127,132,176,182,228,256,267],"273":[19],"expert-crafted":[20],"pronoun":[21],"resolution":[22],"problems":[23],"originally":[24],"designed":[25,136],"to":[26,83,113,155,250,276],"be":[27,251],"unsolvable":[28],"statistical":[30],"models":[31,46,64],"that":[32,81,150,246],"rely":[33,74],"on":[34,53,75,163,179,196,221],"selectional":[35],"preferences":[36],"or":[37,71],"word":[38,153],"associations.":[39,158],"However,":[40],"recent":[41],"advances":[42],"in":[43,78,271],"neural":[44],"language":[45],"have":[47,65,218],"already":[48],"reached":[49],"around":[50],"90%":[51],"accuracy":[52],"variants":[54],"WSC.":[56],"This":[57],"raises":[58],"an":[59,84],"important":[60],"question":[61],"whether":[62,72],"these":[63,261],"truly":[66],"acquired":[67],"robust":[68],"capabilities":[70,89,255],"they":[73,224,242],"spurious":[76],"biases":[77],"the":[79,87,107,116,119,122,128,180,183,226,239,253,265],"datasets":[80],"lead":[82],"overestimation":[85],"true":[88,254],"machine":[91,257],"commonsense.To":[92],"investigate":[93],"this":[94],"question,":[95],"we":[96,191,247],"introduce":[97],"WinoGrande,":[98],"large-scale":[100],"dataset":[101,129],"44k":[103],"problems,":[104],"inspired":[105],"by":[106,140],"original":[108],"WSC":[109,201],"design,":[110],"but":[111],"adjusted":[112],"improve":[114],"both":[115],"scale":[117],"hardness":[120],"dataset.":[123],"key":[125],"steps":[126],"construction":[130],"consist":[131],"(1)":[133],"carefully":[135],"crowdsourcing":[137],"procedure,":[138],"followed":[139],"(2)":[141],"systematic":[142],"bias":[143,269],"reduction":[144,270],"using":[145],"novel":[147],"AfLite":[148],"algorithm":[149],"generalizes":[151],"human-detectable":[152],"associations":[154],"machine-detectable":[156],"embedding":[157],"best":[160],"state-of-the-art":[161,194],"methods":[162],"WinoGrande":[164,229],"achieve":[165],"59.4":[166],"\u2013":[167,188],"79.1%,":[168],"which":[169],"are":[170,248],"\u223c15-35%":[171],"(absolute)":[172],"below":[173],"human":[174],"performance":[175],"94.0%,":[177],"depending":[178],"amount":[181],"training":[184],"data":[185],"allowed":[186],"(2%":[187],"100%":[189],"respectively).Furthermore,":[190],"establish":[192],"new":[193],"results":[195,217],"five":[197],"related":[198],"benchmarks":[199,275],"\u2014":[200],"(\u2192":[202,205,210,214],"90.1%),":[203],"DPR":[204],"93.1%),":[206],"COPA(\u2192":[207],"90.6%),":[208],"KnowRef":[209],"85.6%),":[211],"Winogender":[213],"97.1%).":[215],"These":[216],"dual":[219],"implications:":[220],"one":[222],"hand,":[223,241],"demonstrate":[225],"effectiveness":[227],"when":[230],"used":[231],"as":[232],"resource":[234],"transfer":[236],"learning.":[237],"On":[238],"other":[240],"raise":[243],"concern":[245],"likely":[249],"overestimating":[252],"across":[259],"all":[260],"benchmarks.":[262],"We":[263],"emphasize":[264],"importance":[266],"algorithmic":[268],"existing":[272],"future":[274],"mitigate":[277],"such":[278],"overestimation.":[279]},"counts_by_year":[{"year":2026,"cited_by_count":314},{"year":2025,"cited_by_count":245},{"year":2024,"cited_by_count":142},{"year":2023,"cited_by_count":112},{"year":2022,"cited_by_count":76},{"year":2021,"cited_by_count":101},{"year":2020,"cited_by_count":67},{"year":2019,"cited_by_count":10}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2020-01-10T00:00:00"}