{"id":"https://openalex.org/W2181647815","doi":"https://doi.org/10.1109/acii.2015.7344553","title":"On rater reliability and agreement based dynamic active learning","display_name":"On rater reliability and agreement based dynamic active learning","publication_year":2015,"publication_date":"2015-09-01","ids":{"openalex":"https://openalex.org/W2181647815","doi":"https://doi.org/10.1109/acii.2015.7344553","mag":"2181647815"},"language":"en","primary_location":{"id":"doi:10.1109/acii.2015.7344553","is_oa":false,"landing_page_url":"https://doi.org/10.1109/acii.2015.7344553","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2015 International Conference on Affective Computing and Intelligent Interaction (ACII)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://opus.bibliothek.uni-augsburg.de/opus4/files/72525/72525.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100333755","display_name":"Yue Zhang","orcid":"https://orcid.org/0000-0002-7786-0231"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Yue Zhang","raw_affiliation_strings":["Department of Computing, Imperial College London, London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Department of Computing, Imperial College London, London, United Kingdom","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034958832","display_name":"Eduardo Coutinho","orcid":"https://orcid.org/0000-0001-5234-1497"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Eduardo Coutinho","raw_affiliation_strings":["Department of Computing, Imperial College London, London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Department of Computing, Imperial College London, London, United Kingdom","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043060302","display_name":"Bj\u00f6rn W. Schuller","orcid":"https://orcid.org/0000-0002-6478-8699"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Bjorn Schuller","raw_affiliation_strings":["Department of Computing, Imperial College London, London, United Kingdom","Technische Universitat Munchen, Munchen, Bayern, DE"],"affiliations":[{"raw_affiliation_string":"Department of Computing, Imperial College London, London, United Kingdom","institution_ids":["https://openalex.org/I47508984"]},{"raw_affiliation_string":"Technische Universitat Munchen, Munchen, Bayern, DE","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036056631","display_name":"Zixing Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I186354981","display_name":"University of Passau","ror":"https://ror.org/05ydjnb78","country_code":"DE","type":"education","lineage":["https://openalex.org/I186354981"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Zixing Zhang","raw_affiliation_strings":["Chair of Complex & Intelligent Systems University of Passau, Passau, Germany"],"affiliations":[{"raw_affiliation_string":"Chair of Complex & Intelligent Systems University of Passau, Passau, Germany","institution_ids":["https://openalex.org/I186354981"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043060302","display_name":"Bj\u00f6rn W. Schuller","orcid":"https://orcid.org/0000-0002-6478-8699"},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Bjorn Schuller","raw_affiliation_strings":["Department of Computing, Imperial College London, London, United Kingdom","Technische Universitat Munchen, Munchen, Bayern, DE"],"affiliations":[{"raw_affiliation_string":"Department of Computing, Imperial College London, London, United Kingdom","institution_ids":["https://openalex.org/I47508984"]},{"raw_affiliation_string":"Technische Universitat Munchen, Munchen, Bayern, DE","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102937418","display_name":"Michael G. Adam","orcid":"https://orcid.org/0000-0002-9512-5062"},"institutions":[{"id":"https://openalex.org/I62916508","display_name":"Technical University of Munich","ror":"https://ror.org/02kkvpp62","country_code":"DE","type":"education","lineage":["https://openalex.org/I62916508"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Michael Adam","raw_affiliation_strings":["Machine Intelligence & Signal Processing group, MMK Technische Universit\u00e4t M\u00fcnchen, Munich, Germany"],"affiliations":[{"raw_affiliation_string":"Machine Intelligence & Signal Processing group, MMK Technische Universit\u00e4t M\u00fcnchen, Munich, Germany","institution_ids":["https://openalex.org/I62916508"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100333755"],"corresponding_institution_ids":["https://openalex.org/I47508984"],"apc_list":null,"apc_paid":null,"fwci":4.3144,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.94774221,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"14","issue":null,"first_page":"70","last_page":"76"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9746000170707703,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9739000201225281,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inter-rater-reliability","display_name":"Inter-rater reliability","score":0.6332707405090332},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6066613793373108},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5944174528121948},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.47768908739089966},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.44927629828453064},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.16029968857765198},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.10824990272521973},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.09201884269714355},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09023576974868774}],"concepts":[{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.6332707405090332},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6066613793373108},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5944174528121948},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.47768908739089966},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.44927629828453064},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.16029968857765198},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.10824990272521973},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.09201884269714355},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09023576974868774},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C83849319","wikidata":"https://www.wikidata.org/wiki/Q7295720","display_name":"Rating scale","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/acii.2015.7344553","is_oa":false,"landing_page_url":"https://doi.org/10.1109/acii.2015.7344553","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2015 International Conference on Affective Computing and Intelligent Interaction (ACII)","raw_type":"proceedings-article"},{"id":"pmh:oai:uni-augsburg.opus-bayern.de:72525","is_oa":true,"landing_page_url":"https://nbn-resolving.org/urn:nbn:de:bvb:384-opus4-725253","pdf_url":"https://opus.bibliothek.uni-augsburg.de/opus4/files/72525/72525.pdf","source":{"id":"https://openalex.org/S4306400930","display_name":"OPUS (Augsburg University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I119916105","host_organization_name":"Augsburg University","host_organization_lineage":["https://openalex.org/I119916105"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"bookpart"},{"id":"pmh:oai:mediatum.ub.tum.de:node/1662326","is_oa":false,"landing_page_url":"https://mediatum.ub.tum.de/1662326","pdf_url":null,"source":{"id":"https://openalex.org/S4377196330","display_name":"mediaTUM  (Technical University of Munich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I62916508","host_organization_name":"Technical University of Munich","host_organization_lineage":["https://openalex.org/I62916508"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"ConferencePaper"}],"best_oa_location":{"id":"pmh:oai:uni-augsburg.opus-bayern.de:72525","is_oa":true,"landing_page_url":"https://nbn-resolving.org/urn:nbn:de:bvb:384-opus4-725253","pdf_url":"https://opus.bibliothek.uni-augsburg.de/opus4/files/72525/72525.pdf","source":{"id":"https://openalex.org/S4306400930","display_name":"OPUS (Augsburg University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I119916105","host_organization_name":"Augsburg University","host_organization_lineage":["https://openalex.org/I119916105"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"bookpart"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2181647815.pdf"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W105263016","https://openalex.org/W567437002","https://openalex.org/W1501669607","https://openalex.org/W1618905105","https://openalex.org/W1963882147","https://openalex.org/W2004942499","https://openalex.org/W2023112595","https://openalex.org/W2042700175","https://openalex.org/W2048679005","https://openalex.org/W2055178271","https://openalex.org/W2061863227","https://openalex.org/W2077046587","https://openalex.org/W2083094940","https://openalex.org/W2085662862","https://openalex.org/W2090777335","https://openalex.org/W2124737236","https://openalex.org/W2127816222","https://openalex.org/W2133990480","https://openalex.org/W2136504847","https://openalex.org/W2151401338","https://openalex.org/W2154780170","https://openalex.org/W2161391345","https://openalex.org/W2169027522","https://openalex.org/W2186645669","https://openalex.org/W2398014749","https://openalex.org/W2586921172","https://openalex.org/W2903158431","https://openalex.org/W4234016251","https://openalex.org/W6616005835","https://openalex.org/W6636501900","https://openalex.org/W6678881507","https://openalex.org/W6680140577","https://openalex.org/W6712162632","https://openalex.org/W6756615331","https://openalex.org/W7073743773"],"related_works":["https://openalex.org/W2033512842","https://openalex.org/W4322734194","https://openalex.org/W4233600955","https://openalex.org/W2913665393","https://openalex.org/W2369695847","https://openalex.org/W3005535424","https://openalex.org/W2994319598","https://openalex.org/W2047067935","https://openalex.org/W1607054433","https://openalex.org/W2110842462"],"abstract_inverted_index":{"In":[0,134],"this":[1],"paper,":[2],"we":[3,59,99,140],"propose":[4],"two":[5],"novel":[6],"Dynamic":[7],"Active":[8,34],"Learning":[9,35],"(DAL)":[10],"methods":[11,146],"with":[12],"the":[13,18,38,51,62,101,104,126,131,143,149,152,166,171,181],"aim":[14],"of":[15,53,77,103,151,184],"ultimately":[16],"reducing":[17,157],"costly":[19],"human":[20,89,158],"labelling":[21,159],"work":[22],"for":[23,180],"subjective":[24],"tasks":[25],"such":[26],"as":[27],"speech":[28],"emotion":[29],"recognition.":[30],"Compared":[31],"to":[32,79,93,162],"conventional":[33],"(AL)":[36],"algorithms,":[37],"proposed":[39],"DAL":[40,106,173],"approaches":[41],"employ":[42],"a":[43,74,95,116],"highly":[44,117],"efficient":[45],"adaptive":[46],"query":[47],"strategy":[48],"that":[49,84,142],"minimises":[50],"number":[52,76],"annotations":[54],"through":[55],"three":[56],"advancements.":[57],"First,":[58],"shift":[60],"from":[61],"standard":[63],"majority":[64],"voting":[65],"procedure,":[66],"in":[67],"which":[68],"unlabelled":[69,185],"instances":[70],"are":[71,91],"annotated":[72],"by":[73,108,124,156],"fixed":[75],"raters,":[78],"an":[80],"agreement-based":[81],"annotation":[82],"technique":[83],"dynamically":[85],"determines":[86],"how":[87],"many":[88],"annotators":[90],"required":[92],"label":[94],"selected":[96],"instance.":[97],"Second,":[98],"introduce":[100],"concept":[102],"order-based":[105],"algorithm":[107],"considering":[109],"rater":[110],"reliability":[111],"and":[112],"inter-rater":[113],"agreement.":[114],"Third,":[115],"dynamic":[118,145],"development":[119],"trend":[120],"is":[121],"successfully":[122],"implemented":[123],"upgrading":[125],"agreement":[127],"levels":[128],"depending":[129],"on":[130,137],"prediction":[132],"uncertainty.":[133],"extensive":[135],"experiments":[136],"standardised":[138],"test-beds,":[139],"show":[141],"new":[144],"significantly":[147],"improve":[148],"efficiency":[150],"existing":[153],"AL":[154],"algorithms":[155],"effort":[160],"up":[161,176],"85.41%,":[163],"while":[164],"achieving":[165],"same":[167],"classification":[168],"accuracy.":[169],"Thus,":[170],"enhanced":[172],"derivations":[174],"opens":[175],"high-potential":[177],"research":[178],"directions":[179],"utmost":[182],"exploitation":[183],"data.":[186]},"counts_by_year":[{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}