{"id":"https://openalex.org/W3158434197","doi":"https://doi.org/10.1145/3449236","title":"How do People Train a Machine?","display_name":"How do People Train a Machine?","publication_year":2021,"publication_date":"2021-04-13","ids":{"openalex":"https://openalex.org/W3158434197","doi":"https://doi.org/10.1145/3449236","mag":"3158434197"},"language":"en","primary_location":{"id":"doi:10.1145/3449236","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3449236","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://inria.hal.science/hal-03182950","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018803954","display_name":"T\u00e9o Sanchez","orcid":"https://orcid.org/0000-0001-7221-7020"},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I277688954","display_name":"Universit\u00e9 Paris-Saclay","ror":"https://ror.org/03xjwb503","country_code":"FR","type":"education","lineage":["https://openalex.org/I277688954"]}],"countries":["FR"],"is_corresponding":true,"raw_author_name":"T\u00e9o Sanchez","raw_affiliation_strings":["Universit\u00e9 Paris Saclay, INRIA, LISN, Gif-sur-Yvette, France"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 Paris Saclay, INRIA, LISN, Gif-sur-Yvette, France","institution_ids":["https://openalex.org/I277688954","https://openalex.org/I1326498283"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060825472","display_name":"Baptiste Caramiaux","orcid":"https://orcid.org/0000-0002-4590-106X"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I277688954","display_name":"Universit\u00e9 Paris-Saclay","ror":"https://ror.org/03xjwb503","country_code":"FR","type":"education","lineage":["https://openalex.org/I277688954"]},{"id":"https://openalex.org/I39804081","display_name":"Sorbonne Universit\u00e9","ror":"https://ror.org/02en5vm52","country_code":"FR","type":"education","lineage":["https://openalex.org/I39804081"]},{"id":"https://openalex.org/I4210150358","display_name":"Institut Syst\u00e8mes Intelligents et de Robotique","ror":"https://ror.org/05neq8668","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I154526488","https://openalex.org/I39804081","https://openalex.org/I4210150358","https://openalex.org/I4210159245"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Baptiste Caramiaux","raw_affiliation_strings":["Universit\u00e9 Paris-Saclay, CNRS, Inria, LISN & ISIR & Sorbonne Universit\u00e9, Gif-sur-Yvette, France"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 Paris-Saclay, CNRS, Inria, LISN & ISIR & Sorbonne Universit\u00e9, Gif-sur-Yvette, France","institution_ids":["https://openalex.org/I4210150358","https://openalex.org/I277688954","https://openalex.org/I1294671590","https://openalex.org/I39804081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003198666","display_name":"Jules Fran\u00e7oise","orcid":"https://orcid.org/0000-0003-0461-4010"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I277688954","display_name":"Universit\u00e9 Paris-Saclay","ror":"https://ror.org/03xjwb503","country_code":"FR","type":"education","lineage":["https://openalex.org/I277688954"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Jules Fran\u00e7oise","raw_affiliation_strings":["Universit\u00e9 Paris Saclay, CNRS, LISN, Gif-sur-Yvette, France"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 Paris Saclay, CNRS, LISN, Gif-sur-Yvette, France","institution_ids":["https://openalex.org/I277688954","https://openalex.org/I1294671590"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001141995","display_name":"Fr\u00e9d\u00e9ric Bevilacqua","orcid":"https://orcid.org/0000-0002-2086-3511"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I35345632","display_name":"Institut de Recherche et Coordination Acoustique Musique","ror":"https://ror.org/0121jnt59","country_code":"FR","type":"education","lineage":["https://openalex.org/I35345632"]},{"id":"https://openalex.org/I39804081","display_name":"Sorbonne Universit\u00e9","ror":"https://ror.org/02en5vm52","country_code":"FR","type":"education","lineage":["https://openalex.org/I39804081"]},{"id":"https://openalex.org/I4389425508","display_name":"Sciences et Technologies de la Musique et du Son","ror":"https://ror.org/025xvn046","country_code":null,"type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I2802818602","https://openalex.org/I35345632","https://openalex.org/I39804081","https://openalex.org/I4210102700","https://openalex.org/I4389425508"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Fr\u00e9d\u00e9ric Bevilacqua","raw_affiliation_strings":["STMS IRCAM-CNRS-Sorbonne Universit\u00e9, Paris, France","STMS - Sciences et Technologies de la Musique et du Son (1 Place Igor Stravinsky 75004 PARIS - France)"],"affiliations":[{"raw_affiliation_string":"STMS IRCAM-CNRS-Sorbonne Universit\u00e9, Paris, France","institution_ids":["https://openalex.org/I35345632","https://openalex.org/I39804081","https://openalex.org/I4389425508","https://openalex.org/I1294671590"]},{"raw_affiliation_string":"STMS - Sciences et Technologies de la Musique et du Son (1 Place Igor Stravinsky 75004 PARIS - France)","institution_ids":["https://openalex.org/I35345632","https://openalex.org/I4389425508"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034680150","display_name":"Wendy E. Mackay","orcid":"https://orcid.org/0000-0001-8261-2382"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I277688954","display_name":"Universit\u00e9 Paris-Saclay","ror":"https://ror.org/03xjwb503","country_code":"FR","type":"education","lineage":["https://openalex.org/I277688954"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Wendy E. Mackay","raw_affiliation_strings":["Universit\u00e9 Paris-Saclay, CNRS, Inria, LISN, Gif-sur-Yvette, France"],"affiliations":[{"raw_affiliation_string":"Universit\u00e9 Paris-Saclay, CNRS, Inria, LISN, Gif-sur-Yvette, France","institution_ids":["https://openalex.org/I277688954","https://openalex.org/I1294671590"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5018803954"],"corresponding_institution_ids":["https://openalex.org/I1326498283","https://openalex.org/I277688954"],"apc_list":null,"apc_paid":null,"fwci":2.456,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.89618553,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"5","issue":"CSCW1","first_page":"1","last_page":"26"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10803","display_name":"Innovative Human-Technology Interaction","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11883","display_name":"Embodied and Extended Cognition","score":0.9865999817848206,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sketch","display_name":"Sketch","score":0.9053125977516174},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8040533661842346},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6547865271568298},{"id":"https://openalex.org/keywords/agency","display_name":"Agency (philosophy)","score":0.5409932732582092},{"id":"https://openalex.org/keywords/think-aloud-protocol","display_name":"Think aloud protocol","score":0.49807286262512207},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4797031581401825},{"id":"https://openalex.org/keywords/protocol-analysis","display_name":"Protocol analysis","score":0.4722002446651459},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.46446508169174194},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4560014307498932},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4289408326148987},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3814578354358673},{"id":"https://openalex.org/keywords/cognitive-science","display_name":"Cognitive science","score":0.15332484245300293},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.08190777897834778}],"concepts":[{"id":"https://openalex.org/C2779231336","wikidata":"https://www.wikidata.org/wiki/Q7534724","display_name":"Sketch","level":2,"score":0.9053125977516174},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8040533661842346},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6547865271568298},{"id":"https://openalex.org/C108170787","wikidata":"https://www.wikidata.org/wiki/Q3951828","display_name":"Agency (philosophy)","level":2,"score":0.5409932732582092},{"id":"https://openalex.org/C78821406","wikidata":"https://www.wikidata.org/wiki/Q391810","display_name":"Think aloud protocol","level":3,"score":0.49807286262512207},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4797031581401825},{"id":"https://openalex.org/C133112747","wikidata":"https://www.wikidata.org/wiki/Q7251931","display_name":"Protocol analysis","level":2,"score":0.4722002446651459},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.46446508169174194},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4560014307498932},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4289408326148987},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3814578354358673},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.15332484245300293},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.08190777897834778},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C204787440","wikidata":"https://www.wikidata.org/wiki/Q188504","display_name":"Alternative medicine","level":2,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3449236","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3449236","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},{"id":"pmh:oai:HAL:hal-03182950v1","is_oa":true,"landing_page_url":"https://inria.hal.science/hal-03182950","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"https://cscw.acm.org/2021/","raw_type":"Conference papers"}],"best_oa_location":{"id":"pmh:oai:HAL:hal-03182950v1","is_oa":true,"landing_page_url":"https://inria.hal.science/hal-03182950","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"https://cscw.acm.org/2021/","raw_type":"Conference papers"},"sustainable_development_goals":[{"score":0.6100000143051147,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1389807113","display_name":null,"funder_award_id":"ANR-18-CE33-0002","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G6876561401","display_name":null,"funder_award_id":"ANR-18","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G7758152977","display_name":null,"funder_award_id":"support","funder_id":"https://openalex.org/F4320322892","funder_display_name":"Centre National de la Recherche Scientifique"}],"funders":[{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"},{"id":"https://openalex.org/F4320322892","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":36,"referenced_works":["https://openalex.org/W1501005121","https://openalex.org/W1732926414","https://openalex.org/W1999043044","https://openalex.org/W2003238113","https://openalex.org/W2012508367","https://openalex.org/W2053075547","https://openalex.org/W2059322641","https://openalex.org/W2062427462","https://openalex.org/W2065267227","https://openalex.org/W2084944215","https://openalex.org/W2108598243","https://openalex.org/W2120502006","https://openalex.org/W2121110499","https://openalex.org/W2127058057","https://openalex.org/W2296073425","https://openalex.org/W2345597971","https://openalex.org/W2611748211","https://openalex.org/W2796000004","https://openalex.org/W2798820905","https://openalex.org/W2807910285","https://openalex.org/W2808450727","https://openalex.org/W2898701364","https://openalex.org/W2910392324","https://openalex.org/W2941915890","https://openalex.org/W2943819053","https://openalex.org/W2948288662","https://openalex.org/W2952469525","https://openalex.org/W2963238274","https://openalex.org/W3005126002","https://openalex.org/W3030508559","https://openalex.org/W3032086959","https://openalex.org/W3032875465","https://openalex.org/W3040360103","https://openalex.org/W3106497370","https://openalex.org/W3175803315","https://openalex.org/W4299508536"],"related_works":["https://openalex.org/W4299529040","https://openalex.org/W2188860261","https://openalex.org/W3188118552","https://openalex.org/W2125259630","https://openalex.org/W2016587816","https://openalex.org/W1795122992","https://openalex.org/W2108120676","https://openalex.org/W2127784610","https://openalex.org/W2201589544","https://openalex.org/W2793293581"],"abstract_inverted_index":{"Machine":[0],"learning":[1,38],"systems":[2,170],"became":[3],"pervasive":[4],"in":[5,33,87,114],"modern":[6],"interactive":[7],"technology":[8],"but":[9],"provide":[10],"users":[11],"with":[12,17],"little,":[13],"if":[14],"any,":[15],"agency":[16],"respect":[18],"to":[19,52,172],"how":[20],"their":[21,44],"models":[22],"are":[23,31,152],"trained":[24],"from":[25,43],"data.":[26],"In":[27],"this":[28,180],"paper,":[29],"we":[30,163],"interested":[32],"the":[34,118,123,130,134,176],"way":[35],"novices":[36,173],"handle":[37],"algorithms,":[39],"what":[40,47],"they":[41,49],"understand":[42],"behavior":[45],"and":[46,85,174],"strategy":[48],"may":[50],"use":[51],"\"make":[53],"it":[54],"work\".":[55],"We":[56,76,121,137],"developed":[57],"a":[58,88,103],"web-based":[59],"sketch":[60,126],"recognition":[61],"algorithm":[62],"based":[63],"on":[64,148],"Deep":[65],"Neural":[66],"Network":[67],"(DNN),":[68],"called":[69],"Marcelle-Sketch,":[70],"that":[71,81,109,140],"end-users":[72],"can":[73],"train":[74],"incrementally.":[75],"present":[77],"an":[78],"experimental":[79],"study":[80,93],"investigate":[82],"people's":[83],"strategies":[84,113],"(mis)understandings":[86],"realistic":[89],"algorithm-teaching":[90],"task.":[91,136],"Our":[92,106],"involved":[94],"12":[95],"participants":[96,110],"who":[97],"performed":[98],"individual":[99],"teaching":[100,135],"sessions":[101],"using":[102],"think-aloud":[104],"protocol.":[105],"results":[107],"show":[108],"adopted":[111],"heterogeneous":[112],"which":[115],"variability":[116],"affected":[117],"model":[119],"performances.":[120],"highlighted":[122],"importance":[124],"of":[125,133,158,168,179],"sequencing,":[127],"particularly":[128],"at":[129],"early":[131],"stage":[132],"also":[138],"found":[139],"users'":[141],"understanding":[142],"is":[143],"facilitated":[144],"by":[145,154],"simple":[146],"operations":[147],"drawings,":[149],"while":[150],"confusions":[151],"caused":[153],"certain":[155],"inherent":[156],"properties":[157],"DNN.":[159],"From":[160],"these":[161],"findings,":[162],"propose":[164],"implications":[165],"for":[166],"design":[167],"IML":[169],"dedicated":[171],"discuss":[175],"socio-cultural":[177],"aspect":[178],"research.":[181]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}