{"id":"https://openalex.org/W2799268396","doi":"https://doi.org/10.1145/3184558.3191546","title":"Exploring the Efficiency of Batch Active Learning for Human-in-the-Loop Relation Extraction","display_name":"Exploring the Efficiency of Batch Active Learning for Human-in-the-Loop Relation Extraction","publication_year":2018,"publication_date":"2018-01-01","ids":{"openalex":"https://openalex.org/W2799268396","doi":"https://doi.org/10.1145/3184558.3191546","mag":"2799268396"},"language":"en","primary_location":{"id":"doi:10.1145/3184558.3191546","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3184558.3191546","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3191546&type=pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion of the The Web Conference 2018 on The Web Conference 2018 - WWW '18","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"http://dl.acm.org/ft_gateway.cfm?id=3191546&type=pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043962698","display_name":"Ismini Lourentzou","orcid":"https://orcid.org/0000-0002-1238-772X"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ismini Lourentzou","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006397816","display_name":"Daniel Gruhl","orcid":"https://orcid.org/0009-0006-9124-6752"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel Gruhl","raw_affiliation_strings":["IBM Watson Research Lab, NY, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Watson Research Lab, NY, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108491986","display_name":"Steve Welch","orcid":null},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Steve Welch","raw_affiliation_strings":["IBM Watson Research Lab, NY, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Watson Research Lab, NY, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.6791,"has_fulltext":true,"cited_by_count":15,"citation_normalized_percentile":{"value":0.74174611,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1131","last_page":"1138"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7990456819534302},{"id":"https://openalex.org/keywords/relationship-extraction","display_name":"Relationship extraction","score":0.6793534159660339},{"id":"https://openalex.org/keywords/relation","display_name":"Relation (database)","score":0.6377248764038086},{"id":"https://openalex.org/keywords/active-learning","display_name":"Active learning (machine learning)","score":0.602391242980957},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5501535534858704},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5336889028549194},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5104626417160034},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4941554367542267},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.468193382024765},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.46232298016548157},{"id":"https://openalex.org/keywords/semi-supervised-learning","display_name":"Semi-supervised learning","score":0.4538624584674835},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.4236016869544983},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.34609419107437134},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10266581177711487}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7990456819534302},{"id":"https://openalex.org/C153604712","wikidata":"https://www.wikidata.org/wiki/Q7310755","display_name":"Relationship extraction","level":3,"score":0.6793534159660339},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.6377248764038086},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.602391242980957},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5501535534858704},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5336889028549194},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5104626417160034},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4941554367542267},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.468193382024765},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.46232298016548157},{"id":"https://openalex.org/C58973888","wikidata":"https://www.wikidata.org/wiki/Q1041418","display_name":"Semi-supervised learning","level":2,"score":0.4538624584674835},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.4236016869544983},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.34609419107437134},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10266581177711487},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3184558.3191546","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3184558.3191546","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3191546&type=pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion of the The Web Conference 2018 on The Web Conference 2018 - WWW '18","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3184558.3191546","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3184558.3191546","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=3191546&type=pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion of the The Web Conference 2018 on The Web Conference 2018 - WWW '18","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2799268396.pdf","grobid_xml":"https://content.openalex.org/works/W2799268396.grobid-xml"},"referenced_works_count":61,"referenced_works":["https://openalex.org/W143521272","https://openalex.org/W603830301","https://openalex.org/W950074402","https://openalex.org/W1513874326","https://openalex.org/W1522301498","https://openalex.org/W1551842868","https://openalex.org/W1665214252","https://openalex.org/W1845402413","https://openalex.org/W1912128066","https://openalex.org/W1991234467","https://openalex.org/W1993202648","https://openalex.org/W2012878613","https://openalex.org/W2023352384","https://openalex.org/W2025566624","https://openalex.org/W2030408698","https://openalex.org/W2053238041","https://openalex.org/W2053906594","https://openalex.org/W2060922297","https://openalex.org/W2067367118","https://openalex.org/W2099779943","https://openalex.org/W2120814856","https://openalex.org/W2136502946","https://openalex.org/W2136566870","https://openalex.org/W2137795521","https://openalex.org/W2138627627","https://openalex.org/W2139573966","https://openalex.org/W2140636749","https://openalex.org/W2146191280","https://openalex.org/W2150588363","https://openalex.org/W2155454737","https://openalex.org/W2162590473","https://openalex.org/W2163362093","https://openalex.org/W2165935464","https://openalex.org/W2229639163","https://openalex.org/W2249946156","https://openalex.org/W2250521169","https://openalex.org/W2250539671","https://openalex.org/W2251014694","https://openalex.org/W2251622960","https://openalex.org/W2251847161","https://openalex.org/W2301241615","https://openalex.org/W2404161646","https://openalex.org/W2511964075","https://openalex.org/W2513378248","https://openalex.org/W2515462165","https://openalex.org/W2538200646","https://openalex.org/W2572908757","https://openalex.org/W2573002734","https://openalex.org/W2597787948","https://openalex.org/W2604610161","https://openalex.org/W2735531870","https://openalex.org/W2738031524","https://openalex.org/W2739746722","https://openalex.org/W2767552317","https://openalex.org/W2785704536","https://openalex.org/W2809631449","https://openalex.org/W2903158431","https://openalex.org/W2950371387","https://openalex.org/W2962785888","https://openalex.org/W2964121744","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W2056314584","https://openalex.org/W1492505081","https://openalex.org/W1986633584","https://openalex.org/W2976807053","https://openalex.org/W184546935","https://openalex.org/W2909115059","https://openalex.org/W2949671220","https://openalex.org/W2943886396","https://openalex.org/W4378501087","https://openalex.org/W3203050224"],"abstract_inverted_index":{"Domain-specific":[0],"relation":[1,145],"extraction":[2],"requires":[3],"training":[4,171],"data":[5],"for":[6,20,141],"supervised":[7],"learning":[8,38],"models,":[9],"and":[10,134],"thus,":[11],"significant":[12],"labeling":[13],"effort.":[14],"Distant":[15],"supervision":[16],"is":[17,93,107],"often":[18,108],"leveraged":[19],"creating":[21],"large":[22],"annotated":[23],"corpora":[24],"however":[25,105],"these":[26],"methods":[27],"require":[28],"handling":[29],"the":[30,34,42,47,90,121,126,135,151,170],"inherent":[31],"noise.":[32],"On":[33],"other":[35],"hand,":[36],"active":[37,103],"approaches":[39],"can":[40,64],"reduce":[41],"annotation":[43],"cost":[44],"by":[45],"selecting":[46],"most":[48],"beneficial":[49],"examples":[50,63,83],"to":[51,55,163],"label":[52],"in":[53,72,76,84,98,114,131,138],"order":[54],"learn":[56],"a":[57,80,94,111,132,164],"good":[58],"model.":[59],"The":[60,87],"choice":[61],"of":[62,82,89,102,128,153],"be":[65],"performed":[66],"sequentially,":[67],"i.e.":[68,78],"select":[69,79],"one":[70],"example":[71],"each":[73,85,139],"iteration,":[74],"or":[75],"batches,":[77],"set":[81],"iteration.":[86],"optimization":[88],"batch":[91,133,156],"size":[92,157],"practical":[95],"problem":[96],"faced":[97],"every":[99],"real-world":[100],"application":[101],"learning,":[104],"it":[106],"treated":[109],"as":[110],"parameter":[112],"decided":[113],"advance.":[115],"In":[116],"this":[117],"work,":[118],"we":[119],"study":[120],"trade-off":[122],"between":[123],"model":[124],"performance,":[125,160],"number":[127],"requested":[129],"labels":[130],"time":[136,172],"spent":[137],"round":[140],"real-time,":[142],"domain":[143],"specific":[144],"extraction.":[146],"Our":[147],"results":[148],"show":[149],"that":[150],"use":[152],"an":[154],"appropriate":[155],"produces":[158],"competitive":[159],"even":[161],"compared":[162],"fully":[165],"sequential":[166],"strategy,":[167],"while":[168],"reducing":[169],"dramatically.":[173]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}