{"id":"https://openalex.org/W4366729322","doi":"https://doi.org/10.1145/3544548.3581482","title":"Kaleidoscope: Semantically-grounded, context-specific ML model evaluation","display_name":"Kaleidoscope: Semantically-grounded, context-specific ML model evaluation","publication_year":2023,"publication_date":"2023-04-19","ids":{"openalex":"https://openalex.org/W4366729322","doi":"https://doi.org/10.1145/3544548.3581482"},"language":"en","primary_location":{"id":"doi:10.1145/3544548.3581482","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3581482","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3581482","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3581482","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005452839","display_name":"Harini Suresh","orcid":"https://orcid.org/0000-0002-9769-4947"},"institutions":[{"id":"https://openalex.org/I126820664","display_name":"Vassar College","ror":"https://ror.org/022x6qg61","country_code":"US","type":"education","lineage":["https://openalex.org/I126820664"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Harini Suresh","raw_affiliation_strings":["CSAIL, MIT, United States"],"affiliations":[{"raw_affiliation_string":"CSAIL, MIT, United States","institution_ids":["https://openalex.org/I126820664"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071778232","display_name":"Divya Shanmugam","orcid":"https://orcid.org/0000-0003-1994-053X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Divya Shanmugam","raw_affiliation_strings":["Clinical and Applied Machine Learning Group, MIT, United States"],"affiliations":[{"raw_affiliation_string":"Clinical and Applied Machine Learning Group, MIT, United States","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083496477","display_name":"Tiffany Chen","orcid":"https://orcid.org/0000-0002-7483-3596"},"institutions":[{"id":"https://openalex.org/I126820664","display_name":"Vassar College","ror":"https://ror.org/022x6qg61","country_code":"US","type":"education","lineage":["https://openalex.org/I126820664"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tiffany Chen","raw_affiliation_strings":["CSAIL, MIT, United States"],"affiliations":[{"raw_affiliation_string":"CSAIL, MIT, United States","institution_ids":["https://openalex.org/I126820664"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059936375","display_name":"Annie G Bryan","orcid":"https://orcid.org/0000-0002-1140-8576"},"institutions":[{"id":"https://openalex.org/I126820664","display_name":"Vassar College","ror":"https://ror.org/022x6qg61","country_code":"US","type":"education","lineage":["https://openalex.org/I126820664"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Annie G Bryan","raw_affiliation_strings":["CSAIL, MIT, United States"],"affiliations":[{"raw_affiliation_string":"CSAIL, MIT, United States","institution_ids":["https://openalex.org/I126820664"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060694111","display_name":"Alexander D\u2019Amour","orcid":"https://orcid.org/0000-0001-7984-3366"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alexander D'Amour","raw_affiliation_strings":["Google Research, United States"],"affiliations":[{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007282049","display_name":"John V. Guttag","orcid":"https://orcid.org/0000-0003-0992-0906"},"institutions":[{"id":"https://openalex.org/I126820664","display_name":"Vassar College","ror":"https://ror.org/022x6qg61","country_code":"US","type":"education","lineage":["https://openalex.org/I126820664"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Guttag","raw_affiliation_strings":["CSAIL, MIT, United States"],"affiliations":[{"raw_affiliation_string":"CSAIL, MIT, United States","institution_ids":["https://openalex.org/I126820664"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077783676","display_name":"Arvind Satyanarayan","orcid":"https://orcid.org/0000-0001-5564-635X"},"institutions":[{"id":"https://openalex.org/I126820664","display_name":"Vassar College","ror":"https://ror.org/022x6qg61","country_code":"US","type":"education","lineage":["https://openalex.org/I126820664"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Arvind Satyanarayan","raw_affiliation_strings":["CSAIL, MIT, United States"],"affiliations":[{"raw_affiliation_string":"CSAIL, MIT, United States","institution_ids":["https://openalex.org/I126820664"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5005452839"],"corresponding_institution_ids":["https://openalex.org/I126820664"],"apc_list":null,"apc_paid":null,"fwci":1.9011,"has_fulltext":true,"cited_by_count":11,"citation_normalized_percentile":{"value":0.88285042,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"13"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9889000058174133,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9853000044822693,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/kaleidoscope","display_name":"Kaleidoscope","score":0.9316786527633667},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7896810173988342},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6309634447097778},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6056129336357117},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.5112967491149902},{"id":"https://openalex.org/keywords/moderation","display_name":"Moderation","score":0.5101864337921143},{"id":"https://openalex.org/keywords/digital-subscriber-line","display_name":"Digital subscriber line","score":0.4865802526473999},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.48500844836235046},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.45095294713974},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.41949164867401123},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3855045735836029},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3599604368209839},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.17167222499847412},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.10052204132080078},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.0814143717288971}],"concepts":[{"id":"https://openalex.org/C2778037017","wikidata":"https://www.wikidata.org/wiki/Q6351981","display_name":"Kaleidoscope","level":2,"score":0.9316786527633667},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7896810173988342},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6309634447097778},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6056129336357117},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5112967491149902},{"id":"https://openalex.org/C93225998","wikidata":"https://www.wikidata.org/wiki/Q1941972","display_name":"Moderation","level":2,"score":0.5101864337921143},{"id":"https://openalex.org/C201374245","wikidata":"https://www.wikidata.org/wiki/Q104534","display_name":"Digital subscriber line","level":2,"score":0.4865802526473999},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.48500844836235046},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.45095294713974},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.41949164867401123},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3855045735836029},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3599604368209839},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.17167222499847412},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.10052204132080078},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0814143717288971},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3544548.3581482","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3581482","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3581482","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:dspace.mit.edu:1721.1/150632","is_oa":true,"landing_page_url":"https://hdl.handle.net/1721.1/150632","pdf_url":"https://dspace.mit.edu/bitstream/1721.1/150632/1/3544548.3581482.pdf","source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ACM|Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"http://purl.org/eprint/type/ConferencePaper"}],"best_oa_location":{"id":"doi:10.1145/3544548.3581482","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3544548.3581482","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3544548.3581482","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.6200000047683716,"display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G4263670859","display_name":null,"funder_award_id":"1900991","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4728703731","display_name":null,"funder_award_id":"A8750-19-2-1000","funder_id":"https://openalex.org/F4320332467","funder_display_name":"U.S. Air Force"},{"id":"https://openalex.org/G7647474342","display_name":null,"funder_award_id":"FA8750-19-2-1000","funder_id":"https://openalex.org/F4320332467","funder_display_name":"U.S. Air Force"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332467","display_name":"U.S. Air Force","ror":"https://ror.org/006gmme17"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4366729322.pdf","grobid_xml":"https://content.openalex.org/works/W4366729322.grobid-xml"},"referenced_works_count":35,"referenced_works":["https://openalex.org/W1593532658","https://openalex.org/W1919652261","https://openalex.org/W2030246490","https://openalex.org/W2141880430","https://openalex.org/W2582311786","https://openalex.org/W2810857251","https://openalex.org/W2891177506","https://openalex.org/W2897042519","https://openalex.org/W2899027170","https://openalex.org/W2920807444","https://openalex.org/W2949858875","https://openalex.org/W2970837303","https://openalex.org/W2979826702","https://openalex.org/W3002972902","https://openalex.org/W3003809373","https://openalex.org/W3005013146","https://openalex.org/W3016970897","https://openalex.org/W3035507081","https://openalex.org/W3100279624","https://openalex.org/W3103934428","https://openalex.org/W3112486745","https://openalex.org/W3120485916","https://openalex.org/W3123497755","https://openalex.org/W3130185945","https://openalex.org/W3134111219","https://openalex.org/W3160037564","https://openalex.org/W3172794097","https://openalex.org/W4221104163","https://openalex.org/W4229447062","https://openalex.org/W4230238155","https://openalex.org/W4249922884","https://openalex.org/W4283168936","https://openalex.org/W4288083803","https://openalex.org/W4290943938","https://openalex.org/W4385573966"],"related_works":["https://openalex.org/W625735094","https://openalex.org/W643403496","https://openalex.org/W211986840","https://openalex.org/W4393093108","https://openalex.org/W2955841925","https://openalex.org/W4388316009","https://openalex.org/W337907267","https://openalex.org/W2884840866","https://openalex.org/W2765217416","https://openalex.org/W2081813227"],"abstract_inverted_index":{"Desired":[0],"model":[1,70,75,104],"behavior":[2,76],"often":[3],"differs":[4],"across":[5,145],"contexts":[6],"(e.g.,":[7],"different":[8],"geographies,":[9],"communities,":[10],"or":[11,72],"institutions),":[12],"but":[13],"there":[14],"is":[15],"little":[16],"infrastructure":[17],"to":[18,23,68,97],"facilitate":[19],"context-specific":[20],"evaluations":[21],"key":[22],"deployment":[24],"decisions":[25],"and":[26,89,118,122],"building":[27],"trust.":[28],"Here,":[29],"we":[30,82,113],"present":[31],"Kaleidoscope,":[32,112],"a":[33,50,54,85,101,106,124,132],"system":[34],"for":[35],"evaluating":[36],"models":[37],"in":[38,74,77],"terms":[39],"of":[40],"user-driven,":[41],"domain-relevant":[42],"concepts.":[43],"Kaleidoscope\u2019s":[44],"iterative":[45],"workflow":[46],"enables":[47],"generalizing":[48],"from":[49],"few":[51],"examples":[52,93],"into":[53],"larger,":[55],"diverse":[56],"set":[57,88],"representing":[58],"an":[59],"important":[60],"concept.":[61],"These":[62],"example":[63,148],"sets":[64],"can":[65],"be":[66,98],"used":[67],"test":[69,90],"outputs":[71],"shifts":[73],"semantically-meaningful":[78],"ways.":[79],"For":[80],"instance,":[81],"might":[83],"construct":[84],"\u201cxenophobic":[86],"comments\u201d":[87],"that":[91,138],"its":[92],"are":[94],"more":[95],"likely":[96],"flagged":[99],"by":[100],"content":[102,133],"moderation":[103,134],"than":[105],"\u201ccivil":[107],"discussion\u201d":[108],"set.":[109],"To":[110],"evaluate":[111],"compare":[114],"it":[115],"against":[116],"template-":[117],"DSL-based":[119],"grouping":[120],"methods,":[121],"conduct":[123],"usability":[125],"study":[126],"with":[127],"13":[128],"Reddit":[129],"users":[130],"testing":[131,144],"model.":[135],"We":[136],"find":[137],"Kaleidoscope":[139],"facilitates":[140],"iterative,":[141],"exploratory":[142],"hypothesis":[143],"diverse,":[146],"conceptually-meaningful":[147],"sets.":[149]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":3}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}