{"id":"https://openalex.org/W2162409443","doi":"https://doi.org/10.1145/2556288.2557238","title":"Structured labeling for facilitating concept evolution in machine learning","display_name":"Structured labeling for facilitating concept evolution in machine learning","alternate_titles":[],"publication_year":2014,"publication_date":"2014-04-26","ids":{"openalex":"https://openalex.org/W2162409443","doi":"https://doi.org/10.1145/2556288.2557238","mag":"2162409443"},"language":"en","primary_location":{"id":"doi:10.1145/2556288.2557238","doi":"https://doi.org/10.1145/2556288.2557238","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2556288.2557238","pdf_url":null,"source":{"id":"https://openalex.org/S4363607743","display_name":"CHI Conference on Human Factors in Computing Systems","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SIGCHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W2162409443"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039255125","display_name":"Todd Kulesza","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Todd Kulesza","raw_affiliation_strings":["Oregon State University, Corvallis, OR, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oregon State University, Corvallis, OR, USA","institution_ids":["https://openalex.org/I131249849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057541630","display_name":"Saleema Amershi","orcid":"https://orcid.org/0000-0002-3294-7288","observed_orcids":["https://orcid.org/0000-0002-3294-7288"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Saleema Amershi","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083893060","display_name":"Rich Caruana","orcid":"https://orcid.org/0000-0002-6383-7786","observed_orcids":["https://orcid.org/0000-0002-6383-7786"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rich Caruana","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010157704","display_name":"Danyel Fisher","orcid":"https://orcid.org/0000-0002-9019-5889","observed_orcids":["https://orcid.org/0000-0002-9019-5889"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Danyel Fisher","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008748926","display_name":"Denis Xavier Charles","orcid":"https://orcid.org/0009-0003-7921-3673","observed_orcids":["https://orcid.org/0009-0003-7921-3673"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Denis Charles","raw_affiliation_strings":["Microsoft Research, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.5504,"has_fulltext":false,"cited_by_count":144,"citation_normalized_percentile":{"value":0.9654943,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"3075","last_page":"3084"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.4722000062465668,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.4722000062465668,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.19120000302791595,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.12939999997615814,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/conceptual-evolution","display_name":"conceptual evolution","score":0.8659999966621399},{"id":"https://openalex.org/keywords/human-labeling","display_name":"human labeling","score":0.5040000081062317},{"id":"https://openalex.org/keywords/label-consistency","display_name":"label consistency","score":0.7590000033378601},{"id":"https://openalex.org/keywords/human-in-the-loop-machine-learning","display_name":"human-in-the-loop learning","score":0.6119999885559082},{"id":"https://openalex.org/keywords/spam-email-classification","display_name":"spam email classification","score":0.6069999933242798},{"id":"https://openalex.org/keywords/web-page-classification","display_name":"web page classification","score":0.574999988079071}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7785753011703491},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6254193186759949},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.5890059471130371},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5752329230308533},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.5162467956542969},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4540359675884247},{"id":"https://openalex.org/C60777511","wikidata":"https://www.wikidata.org/wiki/Q3045002","display_name":"Concept drift","level":3,"score":0.42146578431129456},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0798560380935669},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07692626118659973},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C89198739","wikidata":"https://www.wikidata.org/wiki/Q3079880","display_name":"Data stream mining","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2556288.2557238","doi":"https://doi.org/10.1145/2556288.2557238","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2556288.2557238","pdf_url":null,"source":{"id":"https://openalex.org/S4363607743","display_name":"CHI Conference on Human Factors in Computing Systems","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SIGCHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":151,"referenced_works":["https://openalex.org/W27674064","https://openalex.org/W29050389","https://openalex.org/W39957844","https://openalex.org/W79383088","https://openalex.org/W83593200","https://openalex.org/W139044672","https://openalex.org/W204776570","https://openalex.org/W216460935","https://openalex.org/W301677808","https://openalex.org/W303127967","https://openalex.org/W592829405","https://openalex.org/W937107142","https://openalex.org/W945147173","https://openalex.org/W948638995","https://openalex.org/W950482009","https://openalex.org/W952369992","https://openalex.org/W978875813","https://openalex.org/W983732852","https://openalex.org/W993291060","https://openalex.org/W1001401933","https://openalex.org/W1007770941","https://openalex.org/W1011278105","https://openalex.org/W1021027294","https://openalex.org/W1022742273","https://openalex.org/W1026806235","https://openalex.org/W1036507814","https://openalex.org/W1036995229","https://openalex.org/W1483330504","https://openalex.org/W1493108469","https://openalex.org/W1501005121","https://openalex.org/W1502571714","https://openalex.org/W1507264013","https://openalex.org/W1509772896","https://openalex.org/W1510824068","https://openalex.org/W1517680578","https://openalex.org/W1519820146","https://openalex.org/W1520209285","https://openalex.org/W1523794535","https://openalex.org/W1527593353","https://openalex.org/W1530755521","https://openalex.org/W1534753945","https://openalex.org/W1541368278","https://openalex.org/W1546429744","https://openalex.org/W1549909567","https://openalex.org/W1551045851","https://openalex.org/W1554829397","https://openalex.org/W1556861584","https://openalex.org/W1565745825","https://openalex.org/W1567630777","https://openalex.org/W1570169799","https://openalex.org/W1576106211","https://openalex.org/W1576667197","https://openalex.org/W1577085794","https://openalex.org/W1582081223","https://openalex.org/W1582403853","https://openalex.org/W1591119786","https://openalex.org/W1593271214","https://openalex.org/W1598194964","https://openalex.org/W1604542852","https://openalex.org/W1608224733","https://openalex.org/W1655591981","https://openalex.org/W1666942233","https://openalex.org/W1736726159","https://openalex.org/W1748180122","https://openalex.org/W1760018428","https://openalex.org/W1766442844","https://openalex.org/W1812525299","https://openalex.org/W1812816435","https://openalex.org/W1827380967","https://openalex.org/W1837064647","https://openalex.org/W1844936689","https://openalex.org/W1862248568","https://openalex.org/W1867849544","https://openalex.org/W1873332500","https://openalex.org/W1878479415","https://openalex.org/W1881168306","https://openalex.org/W1885089433","https://openalex.org/W1903215186","https://openalex.org/W1922685798","https://openalex.org/W1926997511","https://openalex.org/W1928047241","https://openalex.org/W1937097906","https://openalex.org/W1943883132","https://openalex.org/W1947064154","https://openalex.org/W1952374089","https://openalex.org/W1955441067","https://openalex.org/W1960014134","https://openalex.org/W1963397501","https://openalex.org/W1970736040","https://openalex.org/W2002277714","https://openalex.org/W2022775778","https://openalex.org/W2048987619","https://openalex.org/W2049050102","https://openalex.org/W2062142366","https://openalex.org/W2069324208","https://openalex.org/W2081798681","https://openalex.org/W2112726802","https://openalex.org/W2117664769","https://openalex.org/W2124191450","https://openalex.org/W2125352857","https://openalex.org/W2125943921","https://openalex.org/W2129407262","https://openalex.org/W2135880665","https://openalex.org/W2137799220","https://openalex.org/W2140853685","https://openalex.org/W2142813453","https://openalex.org/W2152311362","https://openalex.org/W2154887314","https://openalex.org/W2157018954","https://openalex.org/W2157504474","https://openalex.org/W2159048649","https://openalex.org/W2161937612","https://openalex.org/W2167708792","https://openalex.org/W2180443860","https://openalex.org/W2200801832","https://openalex.org/W2203634713","https://openalex.org/W2203853577","https://openalex.org/W2206023774","https://openalex.org/W2206651108","https://openalex.org/W2220857793","https://openalex.org/W2235685865","https://openalex.org/W2249399938","https://openalex.org/W2258436308","https://openalex.org/W2264719273","https://openalex.org/W2267992487","https://openalex.org/W2268347781","https://openalex.org/W2268410595","https://openalex.org/W2274531542","https://openalex.org/W2277881186","https://openalex.org/W2279813823","https://openalex.org/W2286544501","https://openalex.org/W2294975956","https://openalex.org/W2298202653","https://openalex.org/W2300607195","https://openalex.org/W2300755140","https://openalex.org/W2306745500","https://openalex.org/W2308569538","https://openalex.org/W2309146138","https://openalex.org/W2396707212","https://openalex.org/W2398872579","https://openalex.org/W2400128484","https://openalex.org/W2401671375","https://openalex.org/W2405804680","https://openalex.org/W2413433895","https://openalex.org/W2414522420","https://openalex.org/W2533923853","https://openalex.org/W2966207845","https://openalex.org/W3141116257","https://openalex.org/W3142582179","https://openalex.org/W3202461034","https://openalex.org/W4235169531"],"related_works":["https://openalex.org/W3120719842","https://openalex.org/W4409884484","https://openalex.org/W4393970668","https://openalex.org/W3000038319","https://openalex.org/W2127779812","https://openalex.org/W4221045135","https://openalex.org/W4385573849","https://openalex.org/W2933315718","https://openalex.org/W2999488406","https://openalex.org/W2117763124"],"abstract_inverted_index":{"Labeling":[0],"data":[1],"is":[2,15,45],"a":[3,32,43,74,88,95,101],"seemingly":[4],"simple":[5],"task":[6],"required":[7],"for":[8,79],"training":[9],"many":[10],"machine":[11,66],"learning":[12],"systems,":[13],"but":[14],"actually":[16],"fraught":[17],"with":[18],"problems.":[19],"This":[20],"paper":[21],"introduces":[22],"the":[23,28,40,107,127],"notion":[24,38],"of":[25,31,39,97,111,129],"concept":[26,35,86,112,130],"evolution,":[27],"changing":[29],"nature":[30],"person's":[33],"underlying":[34],"(the":[36],"abstract":[37],"target":[41],"class":[42],"person":[44],"labeling":[46,72,120],"for,":[47],"e.g.,":[48],"spam":[49],"email,":[50],"travel":[51],"related":[52],"web":[53],"pages)":[54],"which":[55],"can":[56],"result":[57],"in":[58,87,114,126],"inconsistent":[59],"labels":[60],"and":[61,83,109,116],"thus":[62],"be":[63],"detrimental":[64],"to":[65],"learning.":[67],"We":[68],"introduce":[69],"two":[70],"structured":[71,119],"solutions,":[73],"novel":[75],"technique":[76],"we":[77,105],"propose":[78],"helping":[80],"people":[81,122],"define":[82],"refine":[84],"their":[85],"consistent":[89],"manner":[90],"as":[91],"they":[92],"label.":[93],"Through":[94],"series":[96],"five":[98],"experiments,":[99],"including":[100],"controlled":[102],"lab":[103],"study,":[104],"illustrate":[106],"impact":[108],"dynamics":[110],"evolution":[113,131],"practice":[115],"show":[117],"that":[118],"helps":[121],"label":[123],"more":[124],"consistently":[125],"presence":[128],"than":[132],"traditional":[133],"labeling.":[134]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2162409443","counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":12},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":18},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":23},{"year":2018,"cited_by_count":16},{"year":2017,"cited_by_count":11},{"year":2016,"cited_by_count":10},{"year":2015,"cited_by_count":5},{"year":2014,"cited_by_count":2}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2016-06-24T00:00:00"}
