{"id":"https://openalex.org/W2404374285","doi":"https://doi.org/10.1137/1.9781611972818.16","title":"Who Should Label What? Instance Allocation in Multiple Expert Active Learning","display_name":"Who Should Label What? Instance Allocation in Multiple Expert Active Learning","alternate_titles":[],"publication_year":2011,"publication_date":"2011-04-28","ids":{"openalex":"https://openalex.org/W2404374285","doi":"https://doi.org/10.1137/1.9781611972818.16","mag":"2404374285"},"language":"en","primary_location":{"id":"doi:10.1137/1.9781611972818.16","doi":"https://doi.org/10.1137/1.9781611972818.16","is_oa":false,"landing_page_url":"https://doi.org/10.1137/1.9781611972818.16","pdf_url":null,"source":{"id":"https://openalex.org/S4306420871","display_name":"SIAM International Conference on Data Mining","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2011 SIAM International Conference on Data Mining","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W2404374285"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036790226","display_name":"Byron Wallace","orcid":"https://orcid.org/0000-0003-2409-7735","observed_orcids":["https://orcid.org/0000-0003-2409-7735"]},"institutions":[{"id":"https://openalex.org/I121934306","display_name":"Tufts University","ror":"https://ror.org/05wvpxv85","country_code":"US","type":"education","lineage":["https://openalex.org/I121934306"]},{"id":"https://openalex.org/I1290900354","display_name":"Tufts Medical Center","ror":"https://ror.org/002hsbm82","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1290900354"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Byron C. Wallace","raw_affiliation_strings":["Department of Computer Science, Tufts University, Medford, MA, USA Tufts Medical Center, Boston, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USATufts Medical Center, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Tufts University, Medford, MA, USA Tufts Medical Center, Boston, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USATufts Medical Center, Boston, MA, USA","institution_ids":["https://openalex.org/I121934306","https://openalex.org/I1290900354"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090984634","display_name":"Kevin Small","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I121934306","display_name":"Tufts University","ror":"https://ror.org/05wvpxv85","country_code":"US","type":"education","lineage":["https://openalex.org/I121934306"]},{"id":"https://openalex.org/I1290900354","display_name":"Tufts Medical Center","ror":"https://ror.org/002hsbm82","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1290900354"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Kevin Small","raw_affiliation_strings":["Department of Computer Science, Tufts University, Medford, MA, USA Tufts Medical Center, Boston, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USATufts Medical Center, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Tufts University, Medford, MA, USA Tufts Medical Center, Boston, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USATufts Medical Center, Boston, MA, USA","institution_ids":["https://openalex.org/I121934306","https://openalex.org/I1290900354"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045379675","display_name":"Carla E. Brodley","orcid":"https://orcid.org/0009-0008-2134-6285","observed_orcids":["https://orcid.org/0009-0008-2134-6285"]},"institutions":[{"id":"https://openalex.org/I121934306","display_name":"Tufts University","ror":"https://ror.org/05wvpxv85","country_code":"US","type":"education","lineage":["https://openalex.org/I121934306"]},{"id":"https://openalex.org/I1290900354","display_name":"Tufts Medical Center","ror":"https://ror.org/002hsbm82","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1290900354"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Carla E. Brodley","raw_affiliation_strings":["Department of Computer Science, Tufts University, Medford, MA, USA Tufts Medical Center, Boston, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USATufts Medical Center, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Tufts University, Medford, MA, USA Tufts Medical Center, Boston, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USATufts Medical Center, Boston, MA, USA","institution_ids":["https://openalex.org/I121934306","https://openalex.org/I1290900354"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062268434","display_name":"Thomas A Trikalinos","orcid":"https://orcid.org/0000-0002-3990-1848","observed_orcids":["https://orcid.org/0000-0002-3990-1848"]},"institutions":[{"id":"https://openalex.org/I121934306","display_name":"Tufts University","ror":"https://ror.org/05wvpxv85","country_code":"US","type":"education","lineage":["https://openalex.org/I121934306"]},{"id":"https://openalex.org/I1290900354","display_name":"Tufts Medical Center","ror":"https://ror.org/002hsbm82","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1290900354"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Thomas A. Trikalinos","raw_affiliation_strings":["Department of Computer Science, Tufts University, Medford, MA, USA Tufts Medical Center, Boston, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USATufts Medical Center, Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Tufts University, Medford, MA, USA Tufts Medical Center, Boston, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USADepartment of Computer Science, Tufts University, Medford, MA, USATufts Medical Center, Boston, MA, USA","institution_ids":["https://openalex.org/I121934306","https://openalex.org/I1290900354"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.8618,"has_fulltext":false,"cited_by_count":75,"citation_normalized_percentile":{"value":0.96316311,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":86,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"176","last_page":"187"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/label-quality","display_name":"label quality","score":0.6869999766349792},{"id":"https://openalex.org/keywords/metacognition","display_name":"metacognition","score":0.6679999828338623},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"sentiment analysis","score":0.7739999890327454}],"concepts":[{"id":"https://openalex.org/C2779463800","wikidata":"https://www.wikidata.org/wiki/Q5062222","display_name":"Center (category theory)","level":2,"score":0.5294995307922363},{"id":"https://openalex.org/C161191863","wikidata":"https://www.wikidata.org/wiki/Q199655","display_name":"Library science","level":1,"score":0.5111377835273743},{"id":"https://openalex.org/C2993838110","wikidata":"https://www.wikidata.org/wiki/Q494230","display_name":"Medical school","level":2,"score":0.4933101236820221},{"id":"https://openalex.org/C3020610715","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medical science","level":2,"score":0.4539295732975006},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4483717978000641},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3866793215274811},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.2081930935382843},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.1278756558895111},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.08085450530052185},{"id":"https://openalex.org/C8010536","wikidata":"https://www.wikidata.org/wiki/Q160398","display_name":"Crystallography","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1137/1.9781611972818.16","doi":"https://doi.org/10.1137/1.9781611972818.16","is_oa":false,"landing_page_url":"https://doi.org/10.1137/1.9781611972818.16","pdf_url":null,"source":{"id":"https://openalex.org/S4306420871","display_name":"SIAM International Conference on Data Mining","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2011 SIAM International Conference on Data Mining","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.231.4079","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.231.4079","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://siam.omnibooksonline.com/2011datamining/data/papers/099.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W96169002","https://openalex.org/W1522553155","https://openalex.org/W1592853229","https://openalex.org/W1702205140","https://openalex.org/W1976316416","https://openalex.org/W2010135967","https://openalex.org/W2037771830","https://openalex.org/W2043566294","https://openalex.org/W2045030989","https://openalex.org/W2054729949","https://openalex.org/W2055617685","https://openalex.org/W2067760738","https://openalex.org/W2102417402","https://openalex.org/W2105897558","https://openalex.org/W2106897586","https://openalex.org/W2114524997","https://openalex.org/W2125943921","https://openalex.org/W2137935418","https://openalex.org/W2142518823","https://openalex.org/W2146006411","https://openalex.org/W2421229901","https://openalex.org/W2903158431","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W2175199219","https://openalex.org/W2181558882","https://openalex.org/W1966799294","https://openalex.org/W2037771830","https://openalex.org/W2121154891","https://openalex.org/W4318621156","https://openalex.org/W3188862374","https://openalex.org/W7099136555","https://openalex.org/W1881742729","https://openalex.org/W770810907"],"abstract_inverted_index":{"The":[0],"active":[1,69],"learning":[2,70],"(AL)":[3],"framework":[4],"is":[5],"an":[6],"increasingly":[7],"popular":[8],"strategy":[9,106],"for":[10,49,78],"reducing":[11],"the":[12,39,83,94,98],"amount":[13],"of":[14,59,86,97,126,140],"human":[15],"labeling":[16],"effort":[17],"required":[18],"to":[19,92,111,146],"induce":[20],"a":[21,31,42,75,115],"predictive":[22],"model.":[23],"Most":[24],"work":[25],"in":[26,90],"AL":[27,50],"has":[28],"assumed":[29],"that":[30,81,104,134],"single,":[32],"infallible":[33],"oracle":[34],"provides":[35],"labels":[36,58],"requested":[37],"by":[38],"learner":[40],"at":[41],"fixed":[43],"cost.":[44],"However,":[45],"real-world":[46],"applications":[47],"suitable":[48],"often":[51,138],"include":[52],"multiple":[53,67],"domain":[54],"experts":[55,89],"who":[56],"provide":[57,132],"varying":[60],"cost":[61],"and":[62,73,119],"quality.":[63],"We":[64,102],"explore":[65],"this":[66,105],"expert":[68],"(MEAL)":[71],"scenario":[72],"develop":[74],"novel":[76],"algorithm":[77],"instance":[79],"allocation":[80],"exploits":[82],"meta-cognitive":[84],"abilities":[85],"novice":[87,135],"(cheap)":[88],"order":[91],"make":[93],"best":[95],"use":[96],"experienced":[99],"(expensive)":[100],"annotators.":[101],"demonstrate":[103],"outperforms":[107],"strong":[108],"baseline":[109],"approaches":[110],"MEAL":[112],"on":[113],"both":[114],"sentiment":[116],"analysis":[117],"dataset":[118],"two":[120],"datasets":[121],"from":[122],"our":[123],"motivating":[124],"application":[125],"biomedical":[127],"citation":[128],"screening.":[129],"Furthermore,":[130],"we":[131],"evidence":[133],"labelers":[136],"are":[137,144],"aware":[139],"which":[141],"instances":[142],"they":[143],"likely":[145],"mislabel.":[147]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2404374285","counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":7},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":5},{"year":2016,"cited_by_count":6},{"year":2015,"cited_by_count":8},{"year":2014,"cited_by_count":7},{"year":2013,"cited_by_count":4},{"year":2012,"cited_by_count":10}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2016-06-24T00:00:00"}
