{"id":"https://openalex.org/W2039643849","doi":"https://doi.org/10.1145/1753783.1753784","title":"Confidence-based stopping criteria for active learning for data annotation","display_name":"Confidence-based stopping criteria for active learning for data annotation","publication_year":2010,"publication_date":"2010-04-01","ids":{"openalex":"https://openalex.org/W2039643849","doi":"https://doi.org/10.1145/1753783.1753784","mag":"2039643849"},"language":"en","primary_location":{"id":"doi:10.1145/1753783.1753784","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1753783.1753784","pdf_url":null,"source":{"id":"https://openalex.org/S200945739","display_name":"ACM Transactions on Speech and Language Processing","issn_l":"1550-4875","issn":["1550-4875","1550-4883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Speech and Language Processing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100370155","display_name":"Jingbo Zhu","orcid":"https://orcid.org/0000-0002-6537-7007"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Jingbo Zhu","raw_affiliation_strings":["Northeastern University, China"],"affiliations":[{"raw_affiliation_string":"Northeastern University, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101961245","display_name":"Huizhen Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huizhen Wang","raw_affiliation_strings":["Northeastern University, China"],"affiliations":[{"raw_affiliation_string":"Northeastern University, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060225743","display_name":"Eduard Hovy","orcid":"https://orcid.org/0000-0002-3270-7903"},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]},{"id":"https://openalex.org/I2801204180","display_name":"Marina Del Rey Hospital","ror":"https://ror.org/05wqyfz02","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I2801204180"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eduard Hovy","raw_affiliation_strings":["University of Southern California, Marina del Rey, CA"],"affiliations":[{"raw_affiliation_string":"University of Southern California, Marina del Rey, CA","institution_ids":["https://openalex.org/I2801204180","https://openalex.org/I1174212"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081805441","display_name":"Matthew Huei\u2010Ming","orcid":"https://orcid.org/0000-0002-4388-9984"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Ma","raw_affiliation_strings":["Scientific Works, Princeton Junction, NJ","Scientific Works, Princeton Junction, NJ#TAB#"],"affiliations":[{"raw_affiliation_string":"Scientific Works, Princeton Junction, NJ","institution_ids":[]},{"raw_affiliation_string":"Scientific Works, Princeton Junction, NJ#TAB#","institution_ids":["https://openalex.org/I20089843"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5100370155"],"corresponding_institution_ids":["https://openalex.org/I9224756"],"apc_list":null,"apc_paid":null,"fwci":6.0513,"has_fulltext":false,"cited_by_count":87,"citation_normalized_percentile":{"value":0.9605353,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"6","issue":"3","first_page":"1","last_page":"24"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11269","display_name":"Algorithms and Data Compression","score":0.9882000088691711,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.982699990272522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8282734155654907},{"id":"https://openalex.org/keywords/active-learning","display_name":"Active learning (machine learning)","score":0.6571481227874756},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6449965238571167},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6131720542907715},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6101860404014587},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.597873866558075},{"id":"https://openalex.org/keywords/semi-supervised-learning","display_name":"Semi-supervised learning","score":0.5861818790435791},{"id":"https://openalex.org/keywords/early-stopping","display_name":"Early stopping","score":0.5226818919181824},{"id":"https://openalex.org/keywords/supervised-learning","display_name":"Supervised learning","score":0.5050597786903381},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4887807071208954},{"id":"https://openalex.org/keywords/labeled-data","display_name":"Labeled data","score":0.46967512369155884},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.42619845271110535},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3383539915084839}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8282734155654907},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.6571481227874756},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6449965238571167},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6131720542907715},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6101860404014587},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.597873866558075},{"id":"https://openalex.org/C58973888","wikidata":"https://www.wikidata.org/wiki/Q1041418","display_name":"Semi-supervised learning","level":2,"score":0.5861818790435791},{"id":"https://openalex.org/C5465570","wikidata":"https://www.wikidata.org/wiki/Q5326898","display_name":"Early stopping","level":3,"score":0.5226818919181824},{"id":"https://openalex.org/C136389625","wikidata":"https://www.wikidata.org/wiki/Q334384","display_name":"Supervised learning","level":3,"score":0.5050597786903381},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4887807071208954},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.46967512369155884},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.42619845271110535},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3383539915084839},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.0},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/1753783.1753784","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1753783.1753784","pdf_url":null,"source":{"id":"https://openalex.org/S200945739","display_name":"ACM Transactions on Speech and Language Processing","issn_l":"1550-4875","issn":["1550-4875","1550-4883"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Speech and Language Processing","raw_type":"journal-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.300.9824","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.300.9824","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.isi.edu/natural-language/people/hovy/papers/10ACMjournal-activelearning-stopping.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4300000071525574,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G5507929406","display_name":null,"funder_award_id":"60873091","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W108892041","https://openalex.org/W173861811","https://openalex.org/W199218251","https://openalex.org/W401759911","https://openalex.org/W1483816357","https://openalex.org/W1484084878","https://openalex.org/W1490960179","https://openalex.org/W1500592077","https://openalex.org/W1514211892","https://openalex.org/W1514707997","https://openalex.org/W1514940655","https://openalex.org/W1528361845","https://openalex.org/W1549658602","https://openalex.org/W1550206324","https://openalex.org/W1553262910","https://openalex.org/W1583837637","https://openalex.org/W1618905105","https://openalex.org/W1632114991","https://openalex.org/W1968499222","https://openalex.org/W1971731086","https://openalex.org/W1976252502","https://openalex.org/W2021367230","https://openalex.org/W2054804336","https://openalex.org/W2065383075","https://openalex.org/W2065985276","https://openalex.org/W2076083849","https://openalex.org/W2080021732","https://openalex.org/W2085989833","https://openalex.org/W2088911157","https://openalex.org/W2096175520","https://openalex.org/W2103483064","https://openalex.org/W2106894950","https://openalex.org/W2114663556","https://openalex.org/W2117763124","https://openalex.org/W2118108643","https://openalex.org/W2121198199","https://openalex.org/W2139709458","https://openalex.org/W2140679654","https://openalex.org/W2144452292","https://openalex.org/W2151023586","https://openalex.org/W2157025692","https://openalex.org/W2166496241","https://openalex.org/W2168822971","https://openalex.org/W2169027522","https://openalex.org/W2343954916","https://openalex.org/W2346619680","https://openalex.org/W2403007280","https://openalex.org/W2426031434","https://openalex.org/W2531563875","https://openalex.org/W2548695521","https://openalex.org/W2570764145","https://openalex.org/W2886362482","https://openalex.org/W3014245600","https://openalex.org/W3017143921","https://openalex.org/W4285719527","https://openalex.org/W6601048975","https://openalex.org/W6683567821"],"related_works":["https://openalex.org/W2949671220","https://openalex.org/W1492505081","https://openalex.org/W4312414840","https://openalex.org/W2794908468","https://openalex.org/W4206276646","https://openalex.org/W2943467239","https://openalex.org/W1571801203","https://openalex.org/W101422005","https://openalex.org/W192740413","https://openalex.org/W3004135598"],"abstract_inverted_index":{"The":[0],"labor-intensive":[1],"task":[2],"of":[3,57,120,134],"labeling":[4,27],"data":[5,73],"is":[6,41],"a":[7,42,92,96,100,106,121],"serious":[8],"bottleneck":[9],"for":[10,15,29,95,138,150],"many":[11],"supervised":[12,30],"learning":[13,21,31,39,127],"approaches":[14],"natural":[16,153],"language":[17,154],"processing":[18,155],"applications.":[19,49],"Active":[20],"aims":[22],"to":[23,35,90,114],"reduce":[24],"the":[25,37,53,71,110,117,125,132],"human":[26],"cost":[28],"methods.":[32,88],"Determining":[33],"when":[34],"stop":[36],"active":[38,58,126,139],"process":[40],"very":[43],"important":[44],"practical":[45],"issue":[46,56],"in":[47,99],"real-world":[48,148],"This":[50],"article":[51,104],"addresses":[52],"stopping":[54,64,97,122,136],"criterion":[55,98,123,137],"learning,":[59,140],"and":[60,84,164],"presents":[61,105],"four":[62],"simple":[63],"criteria":[65],"based":[66],"on":[67,146],"confidence":[68],"estimation":[69],"over":[70],"unlabeled":[72],"pool,":[74],"including":[75],"maximum":[76],"uncertainty":[77,80],",":[78,81],"overall":[79],"selected":[82],"accuracy,":[83],"minimum":[85],"expected":[86],"error":[87],"Further,":[89],"obtain":[91],"proper":[93],"threshold":[94,119],"specific":[101],"task,":[102],"this":[103],"strategy":[107],"by":[108],"considering":[109],"label":[111],"change":[112],"factor":[113],"dynamically":[115],"update":[116],"predefined":[118],"during":[124],"process.":[128],"To":[129],"empirically":[130],"analyze":[131],"effectiveness":[133],"each":[135],"we":[141],"design":[142],"several":[143],"comparison":[144],"experiments":[145],"seven":[147],"datasets":[149],"three":[151],"representative":[152],"applications":[156],"such":[157],"as":[158],"word":[159],"sense":[160],"disambiguation,":[161],"text":[162],"classification":[163],"opinion":[165],"analysis.":[166]},"counts_by_year":[{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":14},{"year":2021,"cited_by_count":15},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":3},{"year":2015,"cited_by_count":6},{"year":2014,"cited_by_count":2},{"year":2013,"cited_by_count":7},{"year":2012,"cited_by_count":4}],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2025-10-10T00:00:00"}