{"id":"https://openalex.org/W2760672101","doi":"https://doi.org/10.18653/v1/d17-1109","title":"Importance sampling for unbiased on-demand evaluation of knowledge base population","display_name":"Importance sampling for unbiased on-demand evaluation of knowledge base population","publication_year":2017,"publication_date":"2017-01-01","ids":{"openalex":"https://openalex.org/W2760672101","doi":"https://doi.org/10.18653/v1/d17-1109","mag":"2760672101"},"language":"en","primary_location":{"id":"doi:10.18653/v1/d17-1109","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d17-1109","pdf_url":"https://www.aclweb.org/anthology/D17-1109.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 Conference on Empirical Methods in Natural\n          Language Processing","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/D17-1109.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024150057","display_name":"Arun Tejasvi Chaganty","orcid":"https://orcid.org/0000-0001-7122-1298"},"institutions":[{"id":"https://openalex.org/I4210156583","display_name":"Laboratoire d'Informatique de Paris-Nord","ror":"https://ror.org/05g1zjw44","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I4210091279","https://openalex.org/I4210156583","https://openalex.org/I4210159245"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["FR","US"],"is_corresponding":true,"raw_author_name":"Arun Chaganty","raw_affiliation_strings":["Computer Science Department Stanford University"],"affiliations":[{"raw_affiliation_string":"Computer Science Department Stanford University","institution_ids":["https://openalex.org/I4210156583","https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067264969","display_name":"Ashwin Paranjape","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I4210156583","display_name":"Laboratoire d'Informatique de Paris-Nord","ror":"https://ror.org/05g1zjw44","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I4210091279","https://openalex.org/I4210156583","https://openalex.org/I4210159245"]}],"countries":["FR","US"],"is_corresponding":false,"raw_author_name":"Ashwin Paranjape","raw_affiliation_strings":["Computer Science Department Stanford University"],"affiliations":[{"raw_affiliation_string":"Computer Science Department Stanford University","institution_ids":["https://openalex.org/I4210156583","https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025255782","display_name":"Percy Liang","orcid":"https://orcid.org/0000-0002-0458-6139"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I4210156583","display_name":"Laboratoire d'Informatique de Paris-Nord","ror":"https://ror.org/05g1zjw44","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I4210091279","https://openalex.org/I4210156583","https://openalex.org/I4210159245"]}],"countries":["FR","US"],"is_corresponding":false,"raw_author_name":"Percy Liang","raw_affiliation_strings":["Computer Science Department Stanford University"],"affiliations":[{"raw_affiliation_string":"Computer Science Department Stanford University","institution_ids":["https://openalex.org/I4210156583","https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5046006076","display_name":"Christopher D. Manning","orcid":"https://orcid.org/0000-0001-6155-649X"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]},{"id":"https://openalex.org/I4210156583","display_name":"Laboratoire d'Informatique de Paris-Nord","ror":"https://ror.org/05g1zjw44","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I4210091279","https://openalex.org/I4210156583","https://openalex.org/I4210159245"]}],"countries":["FR","US"],"is_corresponding":false,"raw_author_name":"Christopher D. Manning","raw_affiliation_strings":["Computer Science Department Stanford University"],"affiliations":[{"raw_affiliation_string":"Computer Science Department Stanford University","institution_ids":["https://openalex.org/I4210156583","https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5024150057"],"corresponding_institution_ids":["https://openalex.org/I4210156583","https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":1.455,"has_fulltext":true,"cited_by_count":17,"citation_normalized_percentile":{"value":0.87067681,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1038","last_page":"1048"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8631994128227234},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7488903403282166},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.6650027632713318},{"id":"https://openalex.org/keywords/knowledge-base","display_name":"Knowledge base","score":0.5682932138442993},{"id":"https://openalex.org/keywords/population","display_name":"Population","score":0.5634682774543762},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5529214143753052},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.5331330299377441},{"id":"https://openalex.org/keywords/relation","display_name":"Relation (database)","score":0.49322932958602905},{"id":"https://openalex.org/keywords/service","display_name":"Service (business)","score":0.4904293715953827},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.46811550855636597},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.37881046533584595},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.34157758951187134},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3353544771671295},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.11818012595176697},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10334938764572144},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10093757510185242},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.09716048836708069}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8631994128227234},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7488903403282166},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.6650027632713318},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.5682932138442993},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.5634682774543762},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5529214143753052},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.5331330299377441},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.49322932958602905},{"id":"https://openalex.org/C2780378061","wikidata":"https://www.wikidata.org/wiki/Q25351891","display_name":"Service (business)","level":2,"score":0.4904293715953827},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.46811550855636597},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.37881046533584595},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34157758951187134},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3353544771671295},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.11818012595176697},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10334938764572144},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10093757510185242},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.09716048836708069},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C94915269","wikidata":"https://www.wikidata.org/wiki/Q1834857","display_name":"Detector","level":2,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C136264566","wikidata":"https://www.wikidata.org/wiki/Q159810","display_name":"Economy","level":1,"score":0.0},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/d17-1109","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d17-1109","pdf_url":"https://www.aclweb.org/anthology/D17-1109.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 Conference on Empirical Methods in Natural\n          Language Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/d17-1109","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d17-1109","pdf_url":"https://www.aclweb.org/anthology/D17-1109.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 Conference on Empirical Methods in Natural\n          Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.6899999976158142,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G4713059963","display_name":null,"funder_award_id":"FA8750","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2760672101.pdf","grobid_xml":"https://content.openalex.org/works/W2760672101.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W37696276","https://openalex.org/W1507150160","https://openalex.org/W2004918107","https://openalex.org/W2009954908","https://openalex.org/W2057495142","https://openalex.org/W2075893676","https://openalex.org/W2090243146","https://openalex.org/W2102698119","https://openalex.org/W2109244020","https://openalex.org/W2115536324","https://openalex.org/W2123442489","https://openalex.org/W2126170172","https://openalex.org/W2151048449","https://openalex.org/W2181629536","https://openalex.org/W2185226155","https://openalex.org/W2250645263","https://openalex.org/W2251199578","https://openalex.org/W2251847161","https://openalex.org/W2252126225","https://openalex.org/W2252136820","https://openalex.org/W2296128027","https://openalex.org/W2301241615","https://openalex.org/W2469104253","https://openalex.org/W2563621800"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114","https://openalex.org/W4286908577"],"abstract_inverted_index":{"Knowledge":[0],"base":[1],"population":[2],"(KBP)":[3],"systems":[4,136],"take":[5],"in":[6,68,147],"a":[7,40,44,75,87,148],"large":[8],"document":[9],"corpus":[10,142],"and":[11,14,99,143],"extract":[12],"entities":[13],"their":[15],"relations.":[16],"Thus":[17],"far,":[18],"KBP":[19,108,125,140],"evaluation":[20,36,79,126],"has":[21],"relied":[22],"on":[23,25,137],"judgements":[24],"the":[26,69,105,130,138],"pooled":[27],"predictions":[28,90],"of":[29,116],"existing":[30],"systems.":[31],"We":[32,94,128],"show":[33,95],"that":[34],"this":[35,83,96],"is":[37,49,54,74,113],"problematic:":[38],"when":[39],"new":[41,62,76,88],"system":[42],"predicts":[43],"previously":[45],"unseen":[46],"relation,":[47],"it":[48,53],"penalized":[50],"even":[51],"if":[52],"correct.":[55],"This":[56],"leads":[57],"to":[58],"significant":[59],"bias":[60,84,98],"against":[61],"systems,":[63],"which":[64,80],"counterproductively":[65],"discourages":[66],"innovation":[67],"field.":[70],"Our":[71,110],"first":[72],"contribution":[73,112],"importance-sampling":[77],"based":[78],"corrects":[81],"for":[82],"by":[85,132],"annotating":[86],"system's":[89],"ondemand":[91],"via":[92],"crowdsourcing.":[93],"eliminates":[97],"reduces":[100],"variance":[101],"using":[102],"data":[103],"from":[104],"2015":[106],"TAC":[107,139],"task.":[109],"second":[111],"an":[114,123],"implementation":[115],"our":[117],"method":[118],"made":[119],"publicly":[120],"available":[121],"as":[122],"online":[124],"service.":[127],"pilot":[129],"service":[131],"testing":[133],"diverse":[134],"state-ofthe-art":[135],"2016":[141],"obtain":[144],"accurate":[145],"scores":[146],"cost":[149],"effective":[150],"manner.":[151]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":5},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":5}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}