{"id":"https://openalex.org/W4415250162","doi":"https://doi.org/10.1145/3757672","title":"The Hidden Workload: Student Data Work in Multimodal Algorithmic Evaluations","display_name":"The Hidden Workload: Student Data Work in Multimodal Algorithmic Evaluations","publication_year":2025,"publication_date":"2025-10-16","ids":{"openalex":"https://openalex.org/W4415250162","doi":"https://doi.org/10.1145/3757672"},"language":"en","primary_location":{"id":"doi:10.1145/3757672","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3757672","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040730614","display_name":"Gonzalo Gabriel M\u00e9ndez","orcid":"https://orcid.org/0000-0002-3440-1115"},"institutions":[{"id":"https://openalex.org/I60053951","display_name":"Universitat Polit\u00e8cnica de Val\u00e8ncia","ror":"https://ror.org/01460j859","country_code":"ES","type":"education","lineage":["https://openalex.org/I60053951"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"Gonzalo Gabriel M\u00e9ndez","raw_affiliation_strings":["Universitat Polit\u00e8cnica de Val\u00e8ncia, Valencia, Spain and Inria, Rennes, France"],"affiliations":[{"raw_affiliation_string":"Universitat Polit\u00e8cnica de Val\u00e8ncia, Valencia, Spain and Inria, Rennes, France","institution_ids":["https://openalex.org/I60053951"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5117852323","display_name":"Jhonston Hernan Benjumea","orcid":"https://orcid.org/0009-0000-6143-9007"},"institutions":[{"id":"https://openalex.org/I2135383","display_name":"Escuela Superior Politecnica del Litoral","ror":"https://ror.org/04qenc566","country_code":"EC","type":"education","lineage":["https://openalex.org/I2135383"]}],"countries":["EC"],"is_corresponding":false,"raw_author_name":"Jhonston Hernan Benjumea","raw_affiliation_strings":["Escuela Superior Polit\u00e9cnica del Litoral, Guayaquil, Ecuador"],"affiliations":[{"raw_affiliation_string":"Escuela Superior Polit\u00e9cnica del Litoral, Guayaquil, Ecuador","institution_ids":["https://openalex.org/I2135383"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057002286","display_name":"Leonardo Eras","orcid":"https://orcid.org/0000-0002-3594-9289"},"institutions":[{"id":"https://openalex.org/I2135383","display_name":"Escuela Superior Politecnica del Litoral","ror":"https://ror.org/04qenc566","country_code":"EC","type":"education","lineage":["https://openalex.org/I2135383"]}],"countries":["EC"],"is_corresponding":false,"raw_author_name":"Leonardo David Eras","raw_affiliation_strings":["Escuela Superior Polit\u00e9cnica del Litoral, Guayaquil, Ecuador"],"affiliations":[{"raw_affiliation_string":"Escuela Superior Polit\u00e9cnica del Litoral, Guayaquil, Ecuador","institution_ids":["https://openalex.org/I2135383"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052014380","display_name":"Federico Dom\u00ednguez","orcid":"https://orcid.org/0000-0002-3655-2179"},"institutions":[{"id":"https://openalex.org/I2135383","display_name":"Escuela Superior Politecnica del Litoral","ror":"https://ror.org/04qenc566","country_code":"EC","type":"education","lineage":["https://openalex.org/I2135383"]}],"countries":["EC"],"is_corresponding":false,"raw_author_name":"Federico Dom\u00ednguez","raw_affiliation_strings":["Escuela Superior Polit\u00e9cnica del Litoral, Guayaquil, Ecuador"],"affiliations":[{"raw_affiliation_string":"Escuela Superior Polit\u00e9cnica del Litoral, Guayaquil, Ecuador","institution_ids":["https://openalex.org/I2135383"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019858112","display_name":"Marisol Wong-Villacr\u00e9s","orcid":null},"institutions":[{"id":"https://openalex.org/I2135383","display_name":"Escuela Superior Politecnica del Litoral","ror":"https://ror.org/04qenc566","country_code":"EC","type":"education","lineage":["https://openalex.org/I2135383"]}],"countries":["EC"],"is_corresponding":false,"raw_author_name":"Marisol Wong-Villacres","raw_affiliation_strings":["Escuela Superior Polit\u00e9cnica del Litoral, Guayaquil, Ecuador"],"affiliations":[{"raw_affiliation_string":"Escuela Superior Polit\u00e9cnica del Litoral, Guayaquil, Ecuador","institution_ids":["https://openalex.org/I2135383"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5040730614"],"corresponding_institution_ids":["https://openalex.org/I60053951"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.34013455,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"9","issue":"7","first_page":"1","last_page":"30"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9918000102043152,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9803000092506409,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10731","display_name":"Educational Games and Gamification","score":0.9677000045776367,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.7129999995231628},{"id":"https://openalex.org/keywords/autonomy","display_name":"Autonomy","score":0.6105999946594238},{"id":"https://openalex.org/keywords/presentation","display_name":"Presentation (obstetrics)","score":0.5860000252723694},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.5},{"id":"https://openalex.org/keywords/analytics","display_name":"Analytics","score":0.4830000102519989},{"id":"https://openalex.org/keywords/learning-analytics","display_name":"Learning analytics","score":0.45570001006126404},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.4413999915122986},{"id":"https://openalex.org/keywords/data-analysis","display_name":"Data analysis","score":0.3686000108718872}],"concepts":[{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.7129999995231628},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6948000192642212},{"id":"https://openalex.org/C65414064","wikidata":"https://www.wikidata.org/wiki/Q484105","display_name":"Autonomy","level":2,"score":0.6105999946594238},{"id":"https://openalex.org/C2777601897","wikidata":"https://www.wikidata.org/wiki/Q3409113","display_name":"Presentation (obstetrics)","level":2,"score":0.5860000252723694},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5805000066757202},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.5},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.4830000102519989},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.45879998803138733},{"id":"https://openalex.org/C2777648619","wikidata":"https://www.wikidata.org/wiki/Q2845208","display_name":"Learning analytics","level":2,"score":0.45570001006126404},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.4413999915122986},{"id":"https://openalex.org/C175801342","wikidata":"https://www.wikidata.org/wiki/Q1988917","display_name":"Data analysis","level":2,"score":0.3686000108718872},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3490999937057495},{"id":"https://openalex.org/C203481301","wikidata":"https://www.wikidata.org/wiki/Q76221","display_name":"Data system","level":2,"score":0.33070001006126404},{"id":"https://openalex.org/C168031717","wikidata":"https://www.wikidata.org/wiki/Q1530280","display_name":"Balance (ability)","level":2,"score":0.3248000144958496},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.32280001044273376},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.30660000443458557},{"id":"https://openalex.org/C47487241","wikidata":"https://www.wikidata.org/wiki/Q5227230","display_name":"Data access","level":2,"score":0.3052999973297119},{"id":"https://openalex.org/C52970973","wikidata":"https://www.wikidata.org/wiki/Q2497134","display_name":"Adaptive system","level":2,"score":0.3025999963283539},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.2939999997615814},{"id":"https://openalex.org/C2777598771","wikidata":"https://www.wikidata.org/wiki/Q5341279","display_name":"Educational data mining","level":2,"score":0.2906999886035919},{"id":"https://openalex.org/C2776007630","wikidata":"https://www.wikidata.org/wiki/Q2798912","display_name":"Accountability","level":2,"score":0.2838999927043915},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.263700008392334},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.25839999318122864},{"id":"https://openalex.org/C2780198215","wikidata":"https://www.wikidata.org/wiki/Q361428","display_name":"Informal learning","level":2,"score":0.25589999556541443}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3757672","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3757672","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":102,"referenced_works":["https://openalex.org/W627343041","https://openalex.org/W1161474730","https://openalex.org/W1533428769","https://openalex.org/W1542298116","https://openalex.org/W1982464953","https://openalex.org/W1985479696","https://openalex.org/W1986011338","https://openalex.org/W1996730470","https://openalex.org/W2001839578","https://openalex.org/W2021370805","https://openalex.org/W2032406331","https://openalex.org/W2056738644","https://openalex.org/W2068377598","https://openalex.org/W2095270559","https://openalex.org/W2096136396","https://openalex.org/W2101306138","https://openalex.org/W2126192742","https://openalex.org/W2140615673","https://openalex.org/W2164581564","https://openalex.org/W2172044485","https://openalex.org/W2259223404","https://openalex.org/W2263936035","https://openalex.org/W2291875815","https://openalex.org/W2341691066","https://openalex.org/W2400735892","https://openalex.org/W2496197551","https://openalex.org/W2508365783","https://openalex.org/W2530938597","https://openalex.org/W2560359026","https://openalex.org/W2589022383","https://openalex.org/W2593639882","https://openalex.org/W2610281615","https://openalex.org/W2625258984","https://openalex.org/W2738564864","https://openalex.org/W2756291412","https://openalex.org/W2783544451","https://openalex.org/W2791373727","https://openalex.org/W2797620348","https://openalex.org/W2898898463","https://openalex.org/W2899221473","https://openalex.org/W2904980464","https://openalex.org/W2912423735","https://openalex.org/W2941040799","https://openalex.org/W2953526465","https://openalex.org/W2967455432","https://openalex.org/W2985709021","https://openalex.org/W2985936223","https://openalex.org/W2989540417","https://openalex.org/W2990072080","https://openalex.org/W2990407218","https://openalex.org/W3023054078","https://openalex.org/W3028897110","https://openalex.org/W3029417713","https://openalex.org/W3030096167","https://openalex.org/W3036522648","https://openalex.org/W3038651407","https://openalex.org/W3041278472","https://openalex.org/W3046757370","https://openalex.org/W3082311223","https://openalex.org/W3122939459","https://openalex.org/W3135540143","https://openalex.org/W3138001754","https://openalex.org/W3146333390","https://openalex.org/W3151927917","https://openalex.org/W3157605793","https://openalex.org/W3160308348","https://openalex.org/W3162490344","https://openalex.org/W3173535453","https://openalex.org/W3174144098","https://openalex.org/W3190942675","https://openalex.org/W3196912284","https://openalex.org/W3200977999","https://openalex.org/W3206502478","https://openalex.org/W4206205784","https://openalex.org/W4212941576","https://openalex.org/W4213204485","https://openalex.org/W4214588451","https://openalex.org/W4225117352","https://openalex.org/W4225145157","https://openalex.org/W4225162252","https://openalex.org/W4234580964","https://openalex.org/W4280569846","https://openalex.org/W4288083801","https://openalex.org/W4293009012","https://openalex.org/W4294891734","https://openalex.org/W4296371592","https://openalex.org/W4309618422","https://openalex.org/W4323050653","https://openalex.org/W4327990559","https://openalex.org/W4328053313","https://openalex.org/W4361802882","https://openalex.org/W4366595876","https://openalex.org/W4380367290","https://openalex.org/W4381145042","https://openalex.org/W4385338521","https://openalex.org/W4386115159","https://openalex.org/W4386477409","https://openalex.org/W4387344747","https://openalex.org/W4392445448","https://openalex.org/W4392484246","https://openalex.org/W4396232207","https://openalex.org/W4396833135"],"related_works":[],"abstract_inverted_index":{"As":[0],"algorithmic":[1,126],"systems":[2,136],"increasingly":[3],"mediate":[4],"human":[5],"activities":[6],"across":[7],"diverse":[8],"domains,":[9],"they":[10],"shift":[11,33],"more":[12],"responsibility":[13],"for":[14,113,135],"data":[15,24,37,64,87,108],"collection":[16,38,65],"onto":[17],"users,":[18],"fundamentally":[19],"altering":[20],"the":[21,29,97,104,111],"nature":[22],"of":[23,31,85,107],"work.":[25],"This":[26],"paper":[27],"examines":[28],"implications":[30],"this":[32,62],"by":[34],"investigating":[35],"student-led":[36],"and":[39,120,131],"automated":[40],"feedback":[41],"interpretation":[42],"using":[43],"a":[44],"mobile,":[45],"multimodal":[46],"learning":[47],"analytics":[48],"(MMLA)":[49],"tool":[50],"designed":[51],"to":[52,76,80,125,144],"coach":[53],"oral":[54],"presentation":[55],"skills.":[56],"Our":[57],"findings":[58],"reveal":[59],"that":[60,116,137],"while":[61],"user-controlled":[63],"provides":[66],"greater":[67],"flexibility,":[68],"it":[69],"also":[70],"imposes":[71],"speculative":[72],"labor,":[73],"compelling":[74],"students":[75],"adjust":[77],"their":[78],"behavior":[79],"align":[81],"with":[82,141],"perceived":[83],"standards":[84],"good":[86],"even":[88],"when":[89],"such":[90],"changes":[91],"are":[92],"unwarranted.":[93],"The":[94],"study":[95],"highlights":[96],"often-overlooked":[98],"informal":[99],"labor":[100],"involved":[101],"in":[102,129],"managing":[103],"socio-material":[105],"conditions":[106],"collection,":[109],"emphasizing":[110],"need":[112],"MMLA":[114],"tools":[115],"offer":[117],"adaptive":[118],"support":[119],"guidance.":[121],"These":[122],"insights":[123],"extend":[124],"system":[127],"design":[128],"educational":[130],"professional":[132],"contexts,":[133],"advocating":[134],"balance":[138],"user":[139],"autonomy":[140],"workload-minimizing":[142],"guidance":[143],"achieve":[145],"equitable":[146],"accountability.":[147]},"counts_by_year":[],"updated_date":"2026-03-07T16:01:11.037858","created_date":"2025-10-16T00:00:00"}