{"id":"https://openalex.org/W2912924812","doi":"https://doi.org/10.1162/tacl_a_00276","title":"Natural Questions: A Benchmark for Question Answering Research","display_name":"Natural Questions: A Benchmark for Question Answering Research","publication_year":2019,"publication_date":"2019-08-02","ids":{"openalex":"https://openalex.org/W2912924812","doi":"https://doi.org/10.1162/tacl_a_00276","mag":"2912924812"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00276","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00276","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00276/1923288/tacl_a_00276.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00276/1923288/tacl_a_00276.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011150187","display_name":"Tom Kwiatkowski","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Tom Kwiatkowski","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019552678","display_name":"Jennimaria Palomaki","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jennimaria Palomaki","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085764332","display_name":"Olivia Redfield","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Olivia Redfield","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079061237","display_name":"Michael Collins","orcid":"https://orcid.org/0000-0003-0997-1527"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael Collins","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109937799","display_name":"Ankur P. Parikh","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ankur Parikh","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065685274","display_name":"Chris Alberti","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris Alberti","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083857688","display_name":"Danielle Epstein","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Danielle Epstein","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045719436","display_name":"Illia Polosukhin","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Illia Polosukhin","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057457287","display_name":"Jacob Devlin","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jacob Devlin","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081862885","display_name":"Kenton Lee","orcid":"https://orcid.org/0000-0002-9534-5970"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kenton Lee","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053947885","display_name":"Kristina Toutanova","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kristina Toutanova","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023448834","display_name":"Llion Jones","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Llion Jones","raw_affiliation_strings":["Google Research","Google Research, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, Japan","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080258610","display_name":"Matthew Kelcey","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Kelcey","raw_affiliation_strings":["Google Research","Google Research, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, Australia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076904467","display_name":"Ming\u2010Wei Chang","orcid":"https://orcid.org/0000-0002-0137-8895"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ming-Wei Chang","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101597225","display_name":"Andrew M. Dai","orcid":"https://orcid.org/0009-0007-9200-8577"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew M. Dai","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022416424","display_name":"Jakob Uszkoreit","orcid":"https://orcid.org/0000-0001-5066-7530"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jakob Uszkoreit","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088551093","display_name":"Quoc V. Le","orcid":"https://orcid.org/0000-0002-1087-2844"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Quoc Le","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062886906","display_name":"Slav Petrov","orcid":"https://orcid.org/0000-0002-5505-4861"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Slav Petrov","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":18,"corresponding_author_ids":["https://openalex.org/A5011150187"],"corresponding_institution_ids":["https://openalex.org/I1291425158"],"apc_list":null,"apc_paid":null,"fwci":140.244,"has_fulltext":false,"cited_by_count":1968,"citation_normalized_percentile":{"value":0.99977012,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"7","issue":null,"first_page":"453","last_page":"466"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8754678964614868},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.7993302345275879},{"id":"https://openalex.org/keywords/paragraph","display_name":"Paragraph","score":0.7440561652183533},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.7091444134712219},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7039256691932678},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6768658757209778},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6194359660148621},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6027431488037109},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.5112268924713135},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.43941256403923035},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.4389134347438812},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.43240368366241455},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.40674346685409546},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.24287736415863037}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8754678964614868},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.7993302345275879},{"id":"https://openalex.org/C2777206241","wikidata":"https://www.wikidata.org/wiki/Q194431","display_name":"Paragraph","level":2,"score":0.7440561652183533},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7091444134712219},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7039256691932678},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6768658757209778},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6194359660148621},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6027431488037109},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.5112268924713135},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.43941256403923035},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.4389134347438812},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.43240368366241455},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40674346685409546},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.24287736415863037},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/tacl_a_00276","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00276","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00276/1923288/tacl_a_00276.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:8650fdc04d7944c4893d0b995b6de6f7","is_oa":true,"landing_page_url":"https://doaj.org/article/8650fdc04d7944c4893d0b995b6de6f7","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 7, Pp 453-466 (2019)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00276","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00276","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00276/1923288/tacl_a_00276.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.4000000059604645,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2912924812.pdf","grobid_xml":"https://content.openalex.org/works/W2912924812.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W1516184288","https://openalex.org/W1564947197","https://openalex.org/W1593271688","https://openalex.org/W1840435438","https://openalex.org/W2068737686","https://openalex.org/W2101105183","https://openalex.org/W2125436846","https://openalex.org/W2251818205","https://openalex.org/W2413794162","https://openalex.org/W2558203065","https://openalex.org/W2607303097","https://openalex.org/W2888296173","https://openalex.org/W2888302696","https://openalex.org/W2889787757","https://openalex.org/W2890894339","https://openalex.org/W2911430044","https://openalex.org/W2913059114","https://openalex.org/W2949615363","https://openalex.org/W2950501607","https://openalex.org/W2951534261","https://openalex.org/W2962718483","https://openalex.org/W2962809918","https://openalex.org/W2962985038","https://openalex.org/W2963015836","https://openalex.org/W2963323070","https://openalex.org/W2963339397","https://openalex.org/W2963681467","https://openalex.org/W2963748441","https://openalex.org/W2963846996","https://openalex.org/W2963957489","https://openalex.org/W2963963993","https://openalex.org/W2963969878","https://openalex.org/W2964267515","https://openalex.org/W4238634189"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W2377059580"],"abstract_inverted_index":{"We":[0,103,111,130],"present":[1,59,104],"the":[2,20,37,61,109,127,135],"Natural":[3],"Questions":[4,11],"corpus,":[5],"a":[6,29,33,44,48,51,92],"question":[7,30,139],"answering":[8,140],"data":[9],"set.":[10],"consist":[12],"of":[13,76,108,115,137],"real":[14],"anonymized,":[15],"aggregated":[16],"queries":[17],"issued":[18],"to":[19],"Google":[21],"search":[22,40],"engine.":[23],"An":[24],"annotator":[25],"is":[26,70],"presented":[27],"with":[28,32,80,85,96],"along":[31],"Wikipedia":[34],"page":[35],"from":[36,158],"top":[38],"5":[39],"results,":[41],"and":[42,50,91,150],"annotates":[43],"long":[45],"answer":[46,53,69],"(typically":[47],"paragraph)":[49],"short":[52],"(one":[54],"or":[55,63],"more":[56],"entities)":[57],"if":[58,66],"on":[60,118,126,147],"page,":[62],"marks":[64],"null":[65],"no":[67],"long/short":[68],"present.":[71],"The":[72],"public":[73],"release":[74],"consists":[75],"307,373":[77],"training":[78],"examples":[79,84,95],"single":[81],"annotations;":[82],"7,830":[83],"5-way":[86,97],"annotations":[87,117],"for":[88,134],"development":[89],"data;":[90],"further":[93],"7,842":[94],"annotated":[98],"sequestered":[99],"as":[100],"test":[101],"data.":[102,110],"experiments":[105],"validating":[106],"quality":[107],"also":[112],"describe":[113],"analysis":[114],"25-way":[116],"302":[119],"examples,":[120],"giving":[121],"insights":[122],"into":[123],"human":[124,144],"variability":[125],"annotation":[128],"task.":[129],"introduce":[131],"robust":[132],"metrics":[133],"purposes":[136],"evaluating":[138],"systems;":[141],"demonstrate":[142],"high":[143],"upper":[145],"bounds":[146],"these":[148],"metrics;":[149],"establish":[151],"baseline":[152],"results":[153],"using":[154],"competitive":[155],"methods":[156],"drawn":[157],"related":[159],"literature.":[160]},"counts_by_year":[{"year":2026,"cited_by_count":90},{"year":2025,"cited_by_count":320},{"year":2024,"cited_by_count":226},{"year":2023,"cited_by_count":367},{"year":2022,"cited_by_count":304},{"year":2021,"cited_by_count":373},{"year":2020,"cited_by_count":223},{"year":2019,"cited_by_count":65}],"updated_date":"2026-05-05T08:41:31.759640","created_date":"2025-10-10T00:00:00"}