{"id":"https://openalex.org/W2988421999","doi":"https://doi.org/10.18653/v1/d19-5801","title":"MRQA 2019 Shared Task: Evaluating Generalization in Reading Comprehension","display_name":"MRQA 2019 Shared Task: Evaluating Generalization in Reading Comprehension","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2988421999","doi":"https://doi.org/10.18653/v1/d19-5801","mag":"2988421999"},"language":"en","primary_location":{"id":"doi:10.18653/v1/d19-5801","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d19-5801","pdf_url":"https://www.aclweb.org/anthology/D19-5801.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd Workshop on Machine Reading for Question Answering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/D19-5801.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079422282","display_name":"Adam Fisch","orcid":null},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]},{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Adam Fisch","raw_affiliation_strings":["Massachusetts Institute of Technology  Tel Aviv University  Stanford University  University of Washington NAVER  Princeton University","Massachusetts Institute of Technology \u2660 Tel Aviv University \u2663 Stanford University \u2665 University of Washington NAVER \u2665 Princeton University"],"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology  Tel Aviv University  Stanford University  University of Washington NAVER  Princeton University","institution_ids":["https://openalex.org/I20089843"]},{"raw_affiliation_string":"Massachusetts Institute of Technology \u2660 Tel Aviv University \u2663 Stanford University \u2665 University of Washington NAVER \u2665 Princeton University","institution_ids":["https://openalex.org/I97018004","https://openalex.org/I63966007","https://openalex.org/I20089843","https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070878072","display_name":"Alon Talmor","orcid":null},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alon Talmor","raw_affiliation_strings":["Allen Institute for Artificial Intelligence"],"affiliations":[{"raw_affiliation_string":"Allen Institute for Artificial Intelligence","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041906762","display_name":"Robin Jia","orcid":"https://orcid.org/0009-0002-8123-7132"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Robin Jia","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087565126","display_name":"Minjoon Seo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Minjoon Seo","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035142405","display_name":"Eunsol Choi","orcid":"https://orcid.org/0000-0003-3607-9104"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eunsol Choi","raw_affiliation_strings":["Google AI"],"affiliations":[{"raw_affiliation_string":"Google AI","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051064208","display_name":"Danqi Chen","orcid":"https://orcid.org/0000-0002-6226-6838"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Danqi Chen","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5079422282"],"corresponding_institution_ids":["https://openalex.org/I20089843","https://openalex.org/I201448701","https://openalex.org/I63966007","https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":21.0693,"has_fulltext":true,"cited_by_count":212,"citation_normalized_percentile":{"value":0.99501862,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"13"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.821916401386261},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.7371967434883118},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.7365214824676514},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.6590542197227478},{"id":"https://openalex.org/keywords/reading-comprehension","display_name":"Reading comprehension","score":0.6517753005027771},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6356397867202759},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.6101910471916199},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5707967877388},{"id":"https://openalex.org/keywords/reading","display_name":"Reading (process)","score":0.5587455630302429},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.5526806712150574},{"id":"https://openalex.org/keywords/comprehension","display_name":"Comprehension","score":0.5358779430389404},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.43945038318634033},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.0699317455291748},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.06490525603294373},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.06373858451843262}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.821916401386261},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.7371967434883118},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7365214824676514},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.6590542197227478},{"id":"https://openalex.org/C2778780117","wikidata":"https://www.wikidata.org/wiki/Q3269423","display_name":"Reading comprehension","level":3,"score":0.6517753005027771},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6356397867202759},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.6101910471916199},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5707967877388},{"id":"https://openalex.org/C554936623","wikidata":"https://www.wikidata.org/wiki/Q199657","display_name":"Reading (process)","level":2,"score":0.5587455630302429},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.5526806712150574},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.5358779430389404},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43945038318634033},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0699317455291748},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.06490525603294373},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.06373858451843262},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/d19-5801","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d19-5801","pdf_url":"https://www.aclweb.org/anthology/D19-5801.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd Workshop on Machine Reading for Question Answering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/d19-5801","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d19-5801","pdf_url":"https://www.aclweb.org/anthology/D19-5801.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd Workshop on Machine Reading for Question Answering","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.8799999952316284,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320318547","display_name":"Baidu","ror":"https://ror.org/03vs3wt56"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2988421999.pdf","grobid_xml":"https://content.openalex.org/works/W2988421999.grobid-xml"},"referenced_works_count":42,"referenced_works":["https://openalex.org/W107294947","https://openalex.org/W1801866228","https://openalex.org/W1981208470","https://openalex.org/W2028175314","https://openalex.org/W2125436846","https://openalex.org/W2150884987","https://openalex.org/W2252016937","https://openalex.org/W2510759893","https://openalex.org/W2557764419","https://openalex.org/W2606964149","https://openalex.org/W2609826708","https://openalex.org/W2746097825","https://openalex.org/W2799007037","https://openalex.org/W2888302696","https://openalex.org/W2889787757","https://openalex.org/W2896457183","https://openalex.org/W2911681509","https://openalex.org/W2912924812","https://openalex.org/W2919420119","https://openalex.org/W2951831170","https://openalex.org/W2951873305","https://openalex.org/W2953356739","https://openalex.org/W2962881743","https://openalex.org/W2962985038","https://openalex.org/W2963323070","https://openalex.org/W2963339397","https://openalex.org/W2963341956","https://openalex.org/W2963383094","https://openalex.org/W2963748441","https://openalex.org/W2963783335","https://openalex.org/W2963846996","https://openalex.org/W2963928014","https://openalex.org/W2963969878","https://openalex.org/W2964120615","https://openalex.org/W2964223283","https://openalex.org/W2970597249","https://openalex.org/W2984402309","https://openalex.org/W2985436116","https://openalex.org/W2986613042","https://openalex.org/W2986836624","https://openalex.org/W3104486441","https://openalex.org/W4288548690"],"related_works":["https://openalex.org/W207304934","https://openalex.org/W2097810762","https://openalex.org/W2963709474","https://openalex.org/W4288347922","https://openalex.org/W2948810786","https://openalex.org/W2888099120","https://openalex.org/W2970044932","https://openalex.org/W4377703168","https://openalex.org/W2981073299","https://openalex.org/W4382705747"],"abstract_inverted_index":{"We":[0],"present":[1],"the":[2,5,17,38,58,94],"results":[3],"of":[4,20,91],"Machine":[6],"Reading":[7],"for":[8,48,55,63],"Question":[9],"Answering":[10],"(MRQA)":[11],"2019":[12],"shared":[13],"task":[14],"on":[15,93,107],"evaluating":[16],"generalization":[18],"capabilities":[19],"reading":[21],"comprehension":[22],"systems.":[23],"1":[24],"In":[25],"this":[26],"task,":[27],"we":[28],"adapted":[29],"and":[30,57,81],"unified":[31],"18":[32],"distinct":[33],"question":[34],"answering":[35],"datasets":[36,44,51],"into":[37],"same":[39],"format.":[40],"Among":[41],"them,":[42],"six":[43,50,60],"were":[45,52,61],"made":[46,53],"available":[47,54],"training,":[49,80],"development,":[56],"final":[59,64],"hidden":[62],"evaluation.":[65],"Ten":[66],"teams":[67],"submitted":[68],"systems,":[69],"which":[70],"explored":[71],"various":[72],"ideas":[73],"including":[74],"data":[75],"sampling,":[76],"multi-task":[77],"learning,":[78],"adversarial":[79],"ensembling.":[82],"The":[83],"best":[84],"system":[85],"achieved":[86],"an":[87],"average":[88],"F1":[89],"score":[90],"72.5":[92],"12":[95],"held-out":[96],"datasets,":[97],"10.7":[98],"absolute":[99],"points":[100],"higher":[101],"than":[102],"our":[103],"initial":[104],"baseline":[105],"based":[106],"BERT.":[108]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":17},{"year":2023,"cited_by_count":37},{"year":2022,"cited_by_count":45},{"year":2021,"cited_by_count":60},{"year":2020,"cited_by_count":40},{"year":2019,"cited_by_count":1}],"updated_date":"2026-04-02T13:48:15.688549","created_date":"2025-10-10T00:00:00"}