{"id":"https://openalex.org/W2886441967","doi":"https://doi.org/10.18653/v1/w18-2602","title":"Systematic Error Analysis of the Stanford Question Answering Dataset","display_name":"Systematic Error Analysis of the Stanford Question Answering Dataset","publication_year":2018,"publication_date":"2018-01-01","ids":{"openalex":"https://openalex.org/W2886441967","doi":"https://doi.org/10.18653/v1/w18-2602","mag":"2886441967"},"language":"en","primary_location":{"id":"doi:10.18653/v1/w18-2602","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w18-2602","pdf_url":"https://www.aclweb.org/anthology/W18-2602.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Workshop on Machine Reading for Question Answering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/W18-2602.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087734326","display_name":"Marc-Antoine Rondeau","orcid":null},"institutions":[{"id":"https://openalex.org/I4210153468","display_name":"Microsoft (Canada)","ror":"https://ror.org/04xhxg104","country_code":"CA","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210153468"]},{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"id":"https://openalex.org/I4402554038","display_name":"Microsoft Research Montr\u00e9al (Canada)","ror":"https://ror.org/05xdft911","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4402554038"]}],"countries":["CA","GB"],"is_corresponding":false,"raw_author_name":"Marc-Antoine Rondeau","raw_affiliation_strings":["Microsoft Research Montral, Qubec, Canada","Microsoft Research Montr\u00e9al, Qu\u00e9bec, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Montral, Qubec, Canada","institution_ids":["https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Research Montr\u00e9al, Qu\u00e9bec, Canada","institution_ids":["https://openalex.org/I4210153468","https://openalex.org/I4402554038"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5114018312","display_name":"Timothy J. Hazen","orcid":"https://orcid.org/0009-0006-1413-9590"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"T. J. Hazen","raw_affiliation_strings":["Microsoft Research Cambridge, Massachusetts, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Cambridge, Massachusetts, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.0412,"has_fulltext":true,"cited_by_count":31,"citation_normalized_percentile":{"value":0.93188368,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"12","last_page":"20"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9882000088691711,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.8446664214134216},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7573624849319458},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.6903324127197266},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5653015375137329},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5475075244903564},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4583648145198822},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.43268272280693054},{"id":"https://openalex.org/keywords/iterative-and-incremental-development","display_name":"Iterative and incremental development","score":0.4112761914730072},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.34651970863342285},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3462603986263275}],"concepts":[{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.8446664214134216},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7573624849319458},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.6903324127197266},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5653015375137329},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5475075244903564},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4583648145198822},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.43268272280693054},{"id":"https://openalex.org/C143587482","wikidata":"https://www.wikidata.org/wiki/Q1543216","display_name":"Iterative and incremental development","level":2,"score":0.4112761914730072},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.34651970863342285},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3462603986263275},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/w18-2602","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w18-2602","pdf_url":"https://www.aclweb.org/anthology/W18-2602.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Workshop on Machine Reading for Question Answering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/w18-2602","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w18-2602","pdf_url":"https://www.aclweb.org/anthology/W18-2602.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Workshop on Machine Reading for Question Answering","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.46000000834465027,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2886441967.pdf","grobid_xml":"https://content.openalex.org/works/W2886441967.grobid-xml"},"referenced_works_count":15,"referenced_works":["https://openalex.org/W1507711477","https://openalex.org/W2123442489","https://openalex.org/W2521709538","https://openalex.org/W2551396370","https://openalex.org/W2557764419","https://openalex.org/W2600024417","https://openalex.org/W2609826708","https://openalex.org/W2742122443","https://openalex.org/W2750557179","https://openalex.org/W2770970123","https://openalex.org/W2951534261","https://openalex.org/W2963339397","https://openalex.org/W2963748441","https://openalex.org/W2963963993","https://openalex.org/W2963969878"],"related_works":["https://openalex.org/W2805599431","https://openalex.org/W207304934","https://openalex.org/W4377703168","https://openalex.org/W3200179079","https://openalex.org/W2556319748","https://openalex.org/W2123793327","https://openalex.org/W15319282","https://openalex.org/W1560657467","https://openalex.org/W2296730655","https://openalex.org/W2205636404"],"abstract_inverted_index":{"We":[0],"analyzed":[1],"the":[2,12,20,42,71,129,135,140,150,158,165,172],"outputs":[3],"of":[4,41,88,119,154,161],"multiple":[5],"question":[6,146,166],"answering":[7],"(QA)":[8],"models":[9],"applied":[10],"to":[11,18,51,59,64,79,127],"Stanford":[13],"Question":[14],"Answering":[15],"Dataset":[16],"(SQuAD)":[17],"identify":[19,128],"core":[21],"challenges":[22],"for":[23,62,144],"QA":[24,131],"systems":[25,63,138],"on":[26,67,106,116],"this":[27,91],"data":[28],"set.":[29],"Through":[30],"an":[31,86],"iterative":[32],"process,":[33],"challenging":[34],"aspects":[35],"were":[36,57,98],"hypothesized":[37,72],"through":[38],"qualitative":[39],"analysis":[40],"common":[43],"error":[44],"cases.":[45],"A":[46],"classifier":[47],"was":[48,77],"then":[49,148],"constructed":[50],"predict":[52],"whether":[53],"SQuAD":[54],"test":[55],"examples":[56],"likely":[58],"be":[60],"difficult":[61],"answer":[65,142,152],"based":[66,105,115],"features":[68],"associated":[69],"with":[70],"aspects.":[73],"The":[74],"classifier's":[75],"performance":[76],"used":[78],"accept":[80],"or":[81],"reject":[82],"each":[83],"aspect":[84],"as":[85],"indicator":[87],"difficulty.":[89],"With":[90],"approach,":[92],"we":[93],"ensured":[94],"that":[95,155],"our":[96,107],"hypotheses":[97],"systematically":[99],"tested":[100],"and":[101,147],"not":[102,113],"simply":[103],"accepted":[104,114],"pre-existing":[108],"biases.":[109],"Our":[110],"explanations":[111],"are":[112],"human":[117],"evaluation":[118],"individual":[120],"examples.":[121],"This":[122],"process":[123],"also":[124],"enabled":[125],"us":[126],"primary":[130],"strategy":[132],"learned":[133],"by":[134],"models,":[136],"i.e.,":[137],"determined":[139],"acceptable":[141,151],"type":[143,156],"a":[145],"selected":[149],"span":[153],"containing":[157],"highest":[159],"density":[160],"words":[162],"present":[163],"in":[164,171],"within":[167],"its":[168],"local":[169],"vicinity":[170],"passage.":[173]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}