{"id":"https://openalex.org/W3196524514","doi":"https://doi.org/10.1145/3439333","title":"A Taxonomy of Property Measures to Unify Active Learning and Human-centered Approaches to Data Labeling","display_name":"A Taxonomy of Property Measures to Unify Active Learning and Human-centered Approaches to Data Labeling","publication_year":2021,"publication_date":"2021-09-03","ids":{"openalex":"https://openalex.org/W3196524514","doi":"https://doi.org/10.1145/3439333","mag":"3196524514"},"language":"en","primary_location":{"id":"doi:10.1145/3439333","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3439333","pdf_url":null,"source":{"id":"https://openalex.org/S4210173818","display_name":"ACM Transactions on Interactive Intelligent Systems","issn_l":"2160-6455","issn":["2160-6455","2160-6463"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Interactive Intelligent Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078086722","display_name":"J\u00fcrgen Bernard","orcid":"https://orcid.org/0000-0001-8741-9709"},"institutions":[{"id":"https://openalex.org/I141945490","display_name":"University of British Columbia","ror":"https://ror.org/03rmrcq20","country_code":"CA","type":"education","lineage":["https://openalex.org/I141945490"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"J\u00fcrgen Bernard","raw_affiliation_strings":["University of British Columbia, Vancouver, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of British Columbia, Vancouver, Canada","institution_ids":["https://openalex.org/I141945490"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044258783","display_name":"Marco Hutter","orcid":"https://orcid.org/0000-0002-4285-4990"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technische Universit\u00e4t Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Marco Hutter","raw_affiliation_strings":["TU Darmstadt, Darmstadt, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TU Darmstadt, Darmstadt, Germany","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037110552","display_name":"Michael Sedlmair","orcid":"https://orcid.org/0000-0001-7048-9292"},"institutions":[{"id":"https://openalex.org/I100066346","display_name":"University of Stuttgart","ror":"https://ror.org/04vnq7t77","country_code":"DE","type":"education","lineage":["https://openalex.org/I100066346"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Michael Sedlmair","raw_affiliation_strings":["University of Stuttgart, Stuttgart, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Stuttgart, Stuttgart, Germany","institution_ids":["https://openalex.org/I100066346"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060926433","display_name":"Matthias Zeppelzauer","orcid":"https://orcid.org/0000-0003-0413-4746"},"institutions":[{"id":"https://openalex.org/I25485817","display_name":"University of Applied Sciences St P\u00f6lten","ror":"https://ror.org/039a2re55","country_code":"AT","type":"education","lineage":["https://openalex.org/I25485817"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Matthias Zeppelzauer","raw_affiliation_strings":["St. P\u00f6lten University of Applied Sciences, St. P\u00f6lten, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"St. P\u00f6lten University of Applied Sciences, St. P\u00f6lten, Austria","institution_ids":["https://openalex.org/I25485817"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5079916657","display_name":"Tamara Munzner","orcid":"https://orcid.org/0000-0002-3294-3869"},"institutions":[{"id":"https://openalex.org/I141945490","display_name":"University of British Columbia","ror":"https://ror.org/03rmrcq20","country_code":"CA","type":"education","lineage":["https://openalex.org/I141945490"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Tamara Munzner","raw_affiliation_strings":["University of British Columbia, Vancouver, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of British Columbia, Vancouver, Canada","institution_ids":["https://openalex.org/I141945490"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.2248,"has_fulltext":false,"cited_by_count":16,"citation_normalized_percentile":{"value":0.81562038,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"11","issue":"3-4","first_page":"1","last_page":"42"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12205","display_name":"Time Series Analysis and Forecasting","score":0.9732000231742859,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9726999998092651,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7915900349617004},{"id":"https://openalex.org/keywords/visual-analytics","display_name":"Visual analytics","score":0.6063880324363708},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5658718943595886},{"id":"https://openalex.org/keywords/summative-assessment","display_name":"Summative assessment","score":0.5182406902313232},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5179916620254517},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.465853750705719},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4546748101711273},{"id":"https://openalex.org/keywords/analytics","display_name":"Analytics","score":0.44156742095947266},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.4310835897922516},{"id":"https://openalex.org/keywords/property","display_name":"Property (philosophy)","score":0.4151625633239746},{"id":"https://openalex.org/keywords/formative-assessment","display_name":"Formative assessment","score":0.37891435623168945},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.3695661425590515},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3504563570022583},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.11912333965301514}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7915900349617004},{"id":"https://openalex.org/C59732488","wikidata":"https://www.wikidata.org/wiki/Q2528440","display_name":"Visual analytics","level":3,"score":0.6063880324363708},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5658718943595886},{"id":"https://openalex.org/C152747807","wikidata":"https://www.wikidata.org/wiki/Q1854913","display_name":"Summative assessment","level":3,"score":0.5182406902313232},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5179916620254517},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.465853750705719},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4546748101711273},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.44156742095947266},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.4310835897922516},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.4151625633239746},{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.37891435623168945},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.3695661425590515},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3504563570022583},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.11912333965301514},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3439333","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3439333","pdf_url":null,"source":{"id":"https://openalex.org/S4210173818","display_name":"ACM Transactions on Interactive Intelligent Systems","issn_l":"2160-6455","issn":["2160-6455","2160-6463"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Interactive Intelligent Systems","raw_type":"journal-article"},{"id":"pmh:doi:10.5167/uzh-215289","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.6200000047683716}],"awards":[{"id":"https://openalex.org/G7503135576","display_name":null,"funder_award_id":"251654672\u2013TRR 161","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"}],"funders":[{"id":"https://openalex.org/F4320320875","display_name":"Deutscher Akademischer Austauschdienst","ror":"https://ror.org/039djdh30"},{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":125,"referenced_works":["https://openalex.org/W73593153","https://openalex.org/W1494010375","https://openalex.org/W1817561967","https://openalex.org/W1927933524","https://openalex.org/W1928219753","https://openalex.org/W1965555277","https://openalex.org/W1972508834","https://openalex.org/W1974971448","https://openalex.org/W1981039744","https://openalex.org/W1986332411","https://openalex.org/W1987971958","https://openalex.org/W1988732650","https://openalex.org/W1990293817","https://openalex.org/W1992492534","https://openalex.org/W1993202648","https://openalex.org/W1993220166","https://openalex.org/W1995875735","https://openalex.org/W2002008002","https://openalex.org/W2007241592","https://openalex.org/W2011430131","https://openalex.org/W2011434217","https://openalex.org/W2012457675","https://openalex.org/W2016381774","https://openalex.org/W2019557226","https://openalex.org/W2021732807","https://openalex.org/W2029064186","https://openalex.org/W2041277401","https://openalex.org/W2043793719","https://openalex.org/W2048202762","https://openalex.org/W2048305092","https://openalex.org/W2049633694","https://openalex.org/W2051224630","https://openalex.org/W2053724458","https://openalex.org/W2073583237","https://openalex.org/W2080021732","https://openalex.org/W2087194317","https://openalex.org/W2094834818","https://openalex.org/W2097181344","https://openalex.org/W2107131609","https://openalex.org/W2108779687","https://openalex.org/W2109727435","https://openalex.org/W2112179747","https://openalex.org/W2119116964","https://openalex.org/W2119452648","https://openalex.org/W2119821739","https://openalex.org/W2119970716","https://openalex.org/W2122646361","https://openalex.org/W2125775272","https://openalex.org/W2128678390","https://openalex.org/W2137406659","https://openalex.org/W2137983211","https://openalex.org/W2140190241","https://openalex.org/W2141807666","https://openalex.org/W2144182447","https://openalex.org/W2146752106","https://openalex.org/W2152825437","https://openalex.org/W2161816739","https://openalex.org/W2163965546","https://openalex.org/W2164244016","https://openalex.org/W2165232124","https://openalex.org/W2168822971","https://openalex.org/W2170505850","https://openalex.org/W2171671120","https://openalex.org/W2174706414","https://openalex.org/W2187089797","https://openalex.org/W2196365129","https://openalex.org/W2296349981","https://openalex.org/W2332488709","https://openalex.org/W2394669110","https://openalex.org/W2471138382","https://openalex.org/W2482517831","https://openalex.org/W2485962973","https://openalex.org/W2495571292","https://openalex.org/W2510750606","https://openalex.org/W2512274390","https://openalex.org/W2549418531","https://openalex.org/W2562236173","https://openalex.org/W2592594462","https://openalex.org/W2732036031","https://openalex.org/W2739068567","https://openalex.org/W2750714283","https://openalex.org/W2751298778","https://openalex.org/W2752194699","https://openalex.org/W2752257752","https://openalex.org/W2753079413","https://openalex.org/W2753713840","https://openalex.org/W2763367528","https://openalex.org/W2809183397","https://openalex.org/W2824561939","https://openalex.org/W2891503716","https://openalex.org/W2911964244","https://openalex.org/W2913668833","https://openalex.org/W2924477255","https://openalex.org/W2925304665","https://openalex.org/W2935996824","https://openalex.org/W2956281901","https://openalex.org/W2960826148","https://openalex.org/W2963214037","https://openalex.org/W2963715038","https://openalex.org/W2965907370","https://openalex.org/W2969700220","https://openalex.org/W2971280235","https://openalex.org/W2972193535","https://openalex.org/W2975605169","https://openalex.org/W2978934711","https://openalex.org/W2979997102","https://openalex.org/W2999549716","https://openalex.org/W3103496950","https://openalex.org/W3104590849","https://openalex.org/W3105001329","https://openalex.org/W3141164029","https://openalex.org/W3146803896","https://openalex.org/W3148857070","https://openalex.org/W3183522126","https://openalex.org/W4206723194","https://openalex.org/W4213403162","https://openalex.org/W4230737036","https://openalex.org/W4233391551","https://openalex.org/W4238271960","https://openalex.org/W4239510810","https://openalex.org/W4253461361","https://openalex.org/W4285579224","https://openalex.org/W4285719527","https://openalex.org/W6668990524","https://openalex.org/W6761145204"],"related_works":["https://openalex.org/W2596540894","https://openalex.org/W4248485996","https://openalex.org/W1981820048","https://openalex.org/W2373351295","https://openalex.org/W2188168286","https://openalex.org/W4315786423","https://openalex.org/W4323913710","https://openalex.org/W2169821076","https://openalex.org/W2762230188","https://openalex.org/W2410921486"],"abstract_inverted_index":{"Strategies":[0],"for":[1,15,45,163,251,271],"selecting":[2],"the":[3,23,32,85,122,183,205,212,215,218,242,252,256,268,272],"next":[4],"data":[5,14,157],"instance":[6,46,109,261],"to":[7,49,79,107,120,153],"label,":[8],"in":[9,22,31,87,221,277],"service":[10],"of":[11,57,65,71,74,84,118,127,169,173,185,199,234,255,260,274],"generating":[12],"labeled":[13],"machine":[16,24,75,132],"learning,":[17],"have":[18],"been":[19],"considered":[20],"separately":[21],"learning":[25,29,76,133],"literature":[26,35],"on":[27,36],"active":[28],"and":[30,52,125,144,187,194,197,201,217,231,258],"visual":[33,156,160,191],"analytics":[34,161],"human-centered":[37],"approaches.":[38],"We":[39,60,148,209],"propose":[40],"a":[41,62,116,159,227,238,248],"unified":[42],"design":[43],"space":[44],"selection":[47,110,262],"strategies":[48,86,276],"support":[50,121,154],"detailed":[51],"fine-grained":[53,100],"analysis":[54,180,184],"covering":[55,182],"both":[56],"these":[58,88,92],"perspectives.":[59],"identify":[61],"concise":[63],"set":[64],"15":[66],"properties,":[67,93],"namely":[68],"measureable":[69],"characteristics":[70],"datasets":[72,186],"or":[73],"models":[77,189],"applied":[78],"them,":[80],"that":[81,103],"cover":[82],"most":[83],"literatures.":[89],"To":[90],"quantify":[91],"we":[94,114],"introduce":[95],"Property":[96],"Measures":[97],"(PM)":[98],"as":[99,267],"building":[101],"blocks":[102],"can":[104,265],"be":[105],"used":[106],"formalize":[108],"strategies.":[111],"In":[112],"addition,":[113],"present":[115],"taxonomy":[117],"PMs":[119,128,164,170,196],"description,":[123],"evaluation,":[124],"generation":[126],"across":[129],"four":[130],"dimensions:":[131],"(ML)":[134],"Model":[135],"Output":[136],",":[137,140,143],"Instance":[138],"Relations":[139],"Measure":[141,145],"Functionality":[142],"Valence":[146],".":[147],"also":[149],"create":[150],"computational":[151],"infrastructure":[152],"qualitative":[155],"analysis:":[158],"explainer":[162],"built":[165],"around":[166],"an":[167],"implementation":[168],"using":[171,190],"cascades":[172],"eight":[174,179],"atomic":[175],"functions.":[176],"It":[177],"supports":[178],"tasks,":[181],"ML":[188],"comparison":[192],"within":[193],"between":[195],"groups":[198],"PMs,":[200],"over":[202],"time":[203],"during":[204,226],"interactive":[206],"labeling":[207],"process.":[208],"iteratively":[210],"refined":[211],"PM":[213],"taxonomy,":[214],"explainer,":[216],"task":[219],"abstraction":[220],"parallel":[222],"with":[223,241],"each":[224],"other":[225],"two-year":[228],"formative":[229],"process,":[230],"show":[232],"evidence":[233],"their":[235],"utility":[236],"through":[237],"summative":[239],"evaluation":[240],"same":[243],"infrastructure.":[244],"This":[245],"research":[246],"builds":[247],"formal":[249],"baseline":[250],"better":[253],"understanding":[254],"commonalities":[257],"differences":[259],"strategies,":[263],"which":[264],"serve":[266],"stepping":[269],"stone":[270],"synthesis":[273],"novel":[275],"future":[278],"work.":[279]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}