{"id":"https://openalex.org/W2563163303","doi":"https://doi.org/10.18653/v1/d16-1252","title":"Creating a Large Benchmark for Open Information Extraction","display_name":"Creating a Large Benchmark for Open Information Extraction","publication_year":2016,"publication_date":"2016-01-01","ids":{"openalex":"https://openalex.org/W2563163303","doi":"https://doi.org/10.18653/v1/d16-1252","mag":"2563163303"},"language":"en","primary_location":{"id":"doi:10.18653/v1/d16-1252","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d16-1252","pdf_url":"https://www.aclweb.org/anthology/D16-1252.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/D16-1252.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5082136238","display_name":"Gabriel Stanovsky","orcid":"https://orcid.org/0000-0002-2420-8979"},"institutions":[{"id":"https://openalex.org/I13955877","display_name":"Bar-Ilan University","ror":"https://ror.org/03kgsv495","country_code":"IL","type":"education","lineage":["https://openalex.org/I13955877"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Gabriel Stanovsky","raw_affiliation_strings":["Computer Science Department, Bar-Ilan University, Ramat Gan, Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department, Bar-Ilan University, Ramat Gan, Israel","institution_ids":["https://openalex.org/I13955877"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5033006575","display_name":"Ido Dagan","orcid":"https://orcid.org/0000-0003-3415-3374"},"institutions":[{"id":"https://openalex.org/I13955877","display_name":"Bar-Ilan University","ror":"https://ror.org/03kgsv495","country_code":"IL","type":"education","lineage":["https://openalex.org/I13955877"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Ido Dagan","raw_affiliation_strings":["Computer Science Department, Bar-Ilan University, Ramat Gan, Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science Department, Bar-Ilan University, Ramat Gan, Israel","institution_ids":["https://openalex.org/I13955877"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I13955877"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":127,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2300","last_page":"2305"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8284398913383484},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.7874616384506226},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7020114064216614},{"id":"https://openalex.org/keywords/information-extraction","display_name":"Information extraction","score":0.5691788196563721},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5681650042533875},{"id":"https://openalex.org/keywords/implementation","display_name":"Implementation","score":0.5587356686592102},{"id":"https://openalex.org/keywords/open-source","display_name":"Open source","score":0.5483071804046631},{"id":"https://openalex.org/keywords/open-domain","display_name":"Open domain","score":0.5387208461761475},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5134447813034058},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.37914422154426575},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.32478728890419006},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2321673035621643},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.20970410108566284},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.15239578485488892},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.11935362219810486},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.06932204961776733}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8284398913383484},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7874616384506226},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7020114064216614},{"id":"https://openalex.org/C195807954","wikidata":"https://www.wikidata.org/wiki/Q1662562","display_name":"Information extraction","level":2,"score":0.5691788196563721},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5681650042533875},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.5587356686592102},{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.5483071804046631},{"id":"https://openalex.org/C2993776861","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Open domain","level":3,"score":0.5387208461761475},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5134447813034058},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.37914422154426575},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32478728890419006},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2321673035621643},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.20970410108566284},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.15239578485488892},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.11935362219810486},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.06932204961776733},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/d16-1252","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d16-1252","pdf_url":"https://www.aclweb.org/anthology/D16-1252.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/d16-1252","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d16-1252","pdf_url":"https://www.aclweb.org/anthology/D16-1252.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7805275943","display_name":null,"funder_award_id":"DA 1600/1-1","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320322252","display_name":"Israel Science Foundation","ror":"https://ror.org/04sazxf24"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2563163303.pdf","grobid_xml":"https://content.openalex.org/works/W2563163303.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W1493490255","https://openalex.org/W1529731474","https://openalex.org/W2115792525","https://openalex.org/W2126851059","https://openalex.org/W2127978399","https://openalex.org/W2129842875","https://openalex.org/W2152749438","https://openalex.org/W2158847908","https://openalex.org/W2161494021","https://openalex.org/W2167187514","https://openalex.org/W2250386865","https://openalex.org/W2250724656","https://openalex.org/W2251199578","https://openalex.org/W2251687716","https://openalex.org/W2251913848","https://openalex.org/W2252139350","https://openalex.org/W2295434202","https://openalex.org/W2516137589","https://openalex.org/W2915816387"],"related_works":["https://openalex.org/W2469016277","https://openalex.org/W2471366537","https://openalex.org/W2757101400","https://openalex.org/W2369351710","https://openalex.org/W1966454445","https://openalex.org/W1901649692","https://openalex.org/W2726379550","https://openalex.org/W12196170","https://openalex.org/W98961640","https://openalex.org/W3011718088"],"abstract_inverted_index":{"Open":[0,28,41,65,101,114],"information":[1,13],"extraction":[2],"(Open":[3],"IE)":[4],"was":[5],"presented":[6],"as":[7],"an":[8,74],"unrestricted":[9],"variant":[10],"of":[11,26,36,64],"traditional":[12],"extraction.":[14],"It":[15],"has":[16,45],"been":[17,46],"gaining":[18],"substantial":[19],"attention,":[20,39],"manifested":[21],"by":[22],"a":[23,85,95],"large":[24,55,99],"number":[25],"automatic":[27],"IE":[29,42,66,102,115],"extractors":[30],"and":[31,53,76,98,105],"downstream":[32],"applications.":[33],"In":[34,80],"spite":[35],"this":[37,81],"broad":[38],"the":[40,61,89,111],"task":[43],"definition":[44],"lacking":[47],"-there":[48],"are":[49],"no":[50,54],"formal":[51],"guidelines":[52],"scale":[56,70,100],"gold":[57],"standard":[58],"annotation.":[59],"Subsequently,":[60],"various":[62],"implementations":[63],"resorted":[67],"to":[68,93,108],"small":[69],"posthoc":[71],"evaluations,":[72],"inhibiting":[73],"objective":[75],"reproducible":[77],"cross-system":[78],"comparison.":[79],"work,":[82],"we":[83],"develop":[84],"methodology":[86],"that":[87],"leverages":[88],"recent":[90],"QA-SRL":[91],"annotation":[92],"create":[94],"first":[96],"independent":[97],"annotation,":[103],"1":[104],"use":[106],"it":[107],"automatically":[109],"compare":[110],"most":[112],"prominent":[113],"systems.":[116]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":16},{"year":2021,"cited_by_count":20},{"year":2020,"cited_by_count":24},{"year":2019,"cited_by_count":22},{"year":2018,"cited_by_count":19},{"year":2017,"cited_by_count":8}],"updated_date":"2026-07-19T07:52:34.831488","created_date":"2025-10-10T00:00:00"}