{"id":"https://openalex.org/W2963323070","doi":"https://doi.org/10.18653/v1/p18-2124","title":"Know What You Don\u2019t Know: Unanswerable Questions for SQuAD","display_name":"Know What You Don\u2019t Know: Unanswerable Questions for SQuAD","publication_year":2018,"publication_date":"2018-01-01","ids":{"openalex":"https://openalex.org/W2963323070","doi":"https://doi.org/10.18653/v1/p18-2124","mag":"2963323070"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p18-2124","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p18-2124","pdf_url":"https://www.aclweb.org/anthology/P18-2124.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P18-2124.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001094226","display_name":"Pranav Rajpurkar","orcid":"https://orcid.org/0000-0002-8030-3727"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Pranav Rajpurkar","raw_affiliation_strings":["Computer Science Department, Stanford University"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041906762","display_name":"Robin Jia","orcid":"https://orcid.org/0009-0002-8123-7132"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Robin Jia","raw_affiliation_strings":["Computer Science Department, Stanford University"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025255782","display_name":"Percy Liang","orcid":"https://orcid.org/0000-0002-0458-6139"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Percy Liang","raw_affiliation_strings":["Computer Science Department, Stanford University"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Stanford University","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5001094226"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":177.1795,"has_fulltext":true,"cited_by_count":2144,"citation_normalized_percentile":{"value":0.99985532,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"784","last_page":"789"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9921000003814697,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.7411079406738281},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.7198916077613831},{"id":"https://openalex.org/keywords/paragraph","display_name":"Paragraph","score":0.7113741040229797},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7030457258224487},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6248762607574463},{"id":"https://openalex.org/keywords/reading","display_name":"Reading (process)","score":0.5847823023796082},{"id":"https://openalex.org/keywords/successor-cardinal","display_name":"Successor cardinal","score":0.5515846014022827},{"id":"https://openalex.org/keywords/comprehension","display_name":"Comprehension","score":0.5240136384963989},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4995100498199463},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.45696941018104553},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.43091264367103577},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.42950814962387085},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.3229612708091736},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.23693758249282837},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.19082972407341003},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.14878630638122559},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10009202361106873},{"id":"https://openalex.org/keywords/history","display_name":"History","score":0.09561532735824585}],"concepts":[{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.7411079406738281},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.7198916077613831},{"id":"https://openalex.org/C2777206241","wikidata":"https://www.wikidata.org/wiki/Q194431","display_name":"Paragraph","level":2,"score":0.7113741040229797},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7030457258224487},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6248762607574463},{"id":"https://openalex.org/C554936623","wikidata":"https://www.wikidata.org/wiki/Q199657","display_name":"Reading (process)","level":2,"score":0.5847823023796082},{"id":"https://openalex.org/C75306776","wikidata":"https://www.wikidata.org/wiki/Q7632662","display_name":"Successor cardinal","level":2,"score":0.5515846014022827},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.5240136384963989},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4995100498199463},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.45696941018104553},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43091264367103577},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.42950814962387085},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.3229612708091736},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.23693758249282837},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.19082972407341003},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.14878630638122559},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10009202361106873},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.09561532735824585},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/p18-2124","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p18-2124","pdf_url":"https://www.aclweb.org/anthology/P18-2124.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/p18-2124","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p18-2124","pdf_url":"https://www.aclweb.org/anthology/P18-2124.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.6800000071525574}],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2963323070.pdf","grobid_xml":"https://content.openalex.org/works/W2963323070.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W1544827683","https://openalex.org/W1840435438","https://openalex.org/W2118091490","https://openalex.org/W2120735855","https://openalex.org/W2125436846","https://openalex.org/W2240322187","https://openalex.org/W2250790822","https://openalex.org/W2251818205","https://openalex.org/W2510759893","https://openalex.org/W2551396370","https://openalex.org/W2557764419","https://openalex.org/W2558203065","https://openalex.org/W2606964149","https://openalex.org/W2626154462","https://openalex.org/W2740747242","https://openalex.org/W2752104716","https://openalex.org/W2759211898","https://openalex.org/W2770970123","https://openalex.org/W2949615363","https://openalex.org/W2951534261","https://openalex.org/W2951815760","https://openalex.org/W2962718483","https://openalex.org/W2962739339","https://openalex.org/W2962881743","https://openalex.org/W2963339397","https://openalex.org/W2963748441","https://openalex.org/W2963969878","https://openalex.org/W2964298608"],"related_works":["https://openalex.org/W3157284875","https://openalex.org/W2259406085","https://openalex.org/W2099715052","https://openalex.org/W4226247999","https://openalex.org/W4213176082","https://openalex.org/W4220955952","https://openalex.org/W1566366091","https://openalex.org/W3209772662","https://openalex.org/W2908402521","https://openalex.org/W4200629926"],"abstract_inverted_index":{"Extractive":[0],"reading":[1],"comprehension":[2],"systems":[3,96],"can":[4],"often":[5],"locate":[6],"the":[7,29,36,69,113,150,153],"correct":[8,30],"answer":[9,31,100,109],"to":[10,21,55,85,88,149,155],"a":[11,14,64,121,130],"question":[12],"in":[13,35],"context":[15],"document,":[16],"but":[17,104],"they":[18],"also":[19,105],"tend":[20],"make":[22],"unreliable":[23],"guesses":[24],"on":[25,43,94,138,144],"questions":[26,51,80,101],"for":[27,127],"which":[28],"is":[32,110,120],"not":[33,98],"stated":[34],"context.":[37],"Existing":[38],"datasets":[39],"either":[40],"focus":[41],"exclusively":[42],"answerable":[44,89],"questions,":[45],"or":[46],"use":[47],"automatically":[48],"generated":[49],"unanswerable":[50,79],"that":[52,67,134],"are":[53],"easy":[54],"identify.":[56],"To":[57,91],"address":[58],"these":[59],"weaknesses,":[60],"we":[61],"present":[62],"SQUADRUN,":[63,95],"new":[65],"dataset":[66],"combines":[68],"existing":[70,128],"Stanford":[71],"Question":[72],"Answering":[73],"Dataset":[74],"(SQuAD)":[75],"with":[76],"over":[77],"50,000":[78],"written":[81],"adversarially":[82],"by":[83,112],"crowdworkers":[84],"look":[86],"similar":[87],"ones.":[90],"do":[92],"well":[93],"must":[97],"only":[99,141],"when":[102,107],"possible,":[103],"determine":[106],"no":[108],"supported":[111],"paragraph":[114],"and":[115],"abstain":[116],"from":[117],"answering.":[118],"SQUADRUN":[119,148],"challenging":[122],"natural":[123],"language":[124],"understanding":[125],"task":[126],"models:":[129],"strong":[131],"neural":[132],"system":[133],"gets":[135],"86%":[136],"F1":[137,143],"SQuAD":[139],"achieves":[140],"66%":[142],"SQUADRUN.":[145],"We":[146],"release":[147],"community":[151],"as":[152],"successor":[154],"SQuAD.":[156]},"counts_by_year":[{"year":2026,"cited_by_count":31},{"year":2025,"cited_by_count":184},{"year":2024,"cited_by_count":175},{"year":2023,"cited_by_count":392},{"year":2022,"cited_by_count":315},{"year":2021,"cited_by_count":437},{"year":2020,"cited_by_count":372},{"year":2019,"cited_by_count":218},{"year":2018,"cited_by_count":20}],"updated_date":"2026-04-24T08:23:43.765630","created_date":"2025-10-10T00:00:00"}