{"id":"https://openalex.org/W7154190232","doi":"https://doi.org/10.1145/3772318.3790814","title":"EvaluAId: Human-AI Collaborative Evaluation of Open-Ended Student Essays","display_name":"EvaluAId: Human-AI Collaborative Evaluation of Open-Ended Student Essays","publication_year":2026,"publication_date":"2026-04-13","ids":{"openalex":"https://openalex.org/W7154190232","doi":"https://doi.org/10.1145/3772318.3790814"},"language":null,"primary_location":{"id":"doi:10.1145/3772318.3790814","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3790814","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3772318.3790814","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Chao Zhang","orcid":"https://orcid.org/0000-0003-4286-8468"},"institutions":[{"id":"https://openalex.org/I205783295","display_name":"Cornell University","ror":"https://ror.org/05bnh6r87","country_code":"US","type":"education","lineage":["https://openalex.org/I205783295"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Chao Zhang","raw_affiliation_strings":["Cornell University, Ithaca, New York, USA"],"raw_orcid":"https://orcid.org/0000-0003-4286-8468","affiliations":[{"raw_affiliation_string":"Cornell University, Ithaca, New York, USA","institution_ids":["https://openalex.org/I205783295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5117273412","display_name":"Kexin Phyllis Ju","orcid":null},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kexin Phyllis Ju","raw_affiliation_strings":["University of Michigan, Ann Arbor, Michigan, USA"],"raw_orcid":"https://orcid.org/0009-0002-8272-6552","affiliations":[{"raw_affiliation_string":"University of Michigan, Ann Arbor, Michigan, USA","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061577586","display_name":"Xinyi Lu","orcid":"https://orcid.org/0000-0003-2610-6084"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xinyi Lu","raw_affiliation_strings":["Computer Science and Engineering, University of Michigan, Ann Arbor, Michigan, USA"],"raw_orcid":"https://orcid.org/0000-0003-2610-6084","affiliations":[{"raw_affiliation_string":"Computer Science and Engineering, University of Michigan, Ann Arbor, Michigan, USA","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5095927531","display_name":"Yu-Chun Grace Yen","orcid":"https://orcid.org/0000-0002-5442-6934"},"institutions":[{"id":"https://openalex.org/I148366613","display_name":"National Yang Ming Chiao Tung University","ror":"https://ror.org/00se2k293","country_code":"TW","type":"education","lineage":["https://openalex.org/I148366613"]}],"countries":["TW"],"is_corresponding":false,"raw_author_name":"Yu-Chun Grace Yen","raw_affiliation_strings":["Computer Science, National Yang Ming Chiao Tung University, Hsinchu, Taiwan"],"raw_orcid":"https://orcid.org/0000-0002-5442-6934","affiliations":[{"raw_affiliation_string":"Computer Science, National Yang Ming Chiao Tung University, Hsinchu, Taiwan","institution_ids":["https://openalex.org/I148366613"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024501566","display_name":"Jeffrey M. Rzeszotarski","orcid":"https://orcid.org/0000-0002-4317-9501"},"institutions":[{"id":"https://openalex.org/I165556055","display_name":"Loyola University Maryland","ror":"https://ror.org/01by1wp65","country_code":"US","type":"education","lineage":["https://openalex.org/I165556055"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeffrey M. Rzeszotarski","raw_affiliation_strings":["Department of Computer Science, Loyola University Maryland, Baltimore, Maryland, USA"],"raw_orcid":"https://orcid.org/0000-0002-4317-9501","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Loyola University Maryland, Baltimore, Maryland, USA","institution_ids":["https://openalex.org/I165556055"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I205783295"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.85277632,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"20"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1454000025987625,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1454000025987625,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.11659999936819077,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12887","display_name":"Educational Strategies and Epistemologies","score":0.08079999685287476,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/formative-assessment","display_name":"Formative assessment","score":0.7157999873161316},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6039999723434448},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.5751000046730042},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.5439000129699707},{"id":"https://openalex.org/keywords/chatbot","display_name":"Chatbot","score":0.5153999924659729},{"id":"https://openalex.org/keywords/transparency","display_name":"Transparency (behavior)","score":0.49050000309944153},{"id":"https://openalex.org/keywords/online-assessment","display_name":"Online assessment","score":0.3930000066757202},{"id":"https://openalex.org/keywords/higher-education","display_name":"Higher education","score":0.3767000138759613}],"concepts":[{"id":"https://openalex.org/C42525527","wikidata":"https://www.wikidata.org/wiki/Q1209955","display_name":"Formative assessment","level":2,"score":0.7157999873161316},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6039999723434448},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.5751000046730042},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5461999773979187},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.5439000129699707},{"id":"https://openalex.org/C2779041454","wikidata":"https://www.wikidata.org/wiki/Q870780","display_name":"Chatbot","level":2,"score":0.5153999924659729},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.49050000309944153},{"id":"https://openalex.org/C178823583","wikidata":"https://www.wikidata.org/wiki/Q3269883","display_name":"Online assessment","level":3,"score":0.3930000066757202},{"id":"https://openalex.org/C120912362","wikidata":"https://www.wikidata.org/wiki/Q136822","display_name":"Higher education","level":2,"score":0.3767000138759613},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.365200012922287},{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.34880000352859497},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.33899998664855957},{"id":"https://openalex.org/C100609095","wikidata":"https://www.wikidata.org/wiki/Q1335050","display_name":"Embodied cognition","level":2,"score":0.3328999876976013},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.313400000333786},{"id":"https://openalex.org/C55587333","wikidata":"https://www.wikidata.org/wiki/Q1133029","display_name":"Engineering ethics","level":1,"score":0.31279999017715454},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.3095000088214874},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3018999993801117},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.30149999260902405},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2987000048160553},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2971000075340271},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.2847000062465668},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.28110000491142273},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.27889999747276306},{"id":"https://openalex.org/C138020889","wikidata":"https://www.wikidata.org/wiki/Q2349659","display_name":"Collaborative learning","level":2,"score":0.2721000015735626},{"id":"https://openalex.org/C108771440","wikidata":"https://www.wikidata.org/wiki/Q368475","display_name":"Lifelong learning","level":2,"score":0.26930001378059387},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2660999894142151},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.2660999894142151},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.2549999952316284},{"id":"https://openalex.org/C2781118332","wikidata":"https://www.wikidata.org/wiki/Q430460","display_name":"Capability approach","level":2,"score":0.2515999972820282}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3772318.3790814","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3790814","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3772318.3790814","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3772318.3790814","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.8828482031822205,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":80,"referenced_works":["https://openalex.org/W1971011382","https://openalex.org/W1977120130","https://openalex.org/W1979290264","https://openalex.org/W1992861205","https://openalex.org/W2044140042","https://openalex.org/W2060423038","https://openalex.org/W2063052894","https://openalex.org/W2064341783","https://openalex.org/W2067895686","https://openalex.org/W2072593146","https://openalex.org/W2084015037","https://openalex.org/W2093950336","https://openalex.org/W2094218279","https://openalex.org/W2096452841","https://openalex.org/W2109830295","https://openalex.org/W2115389369","https://openalex.org/W2124853884","https://openalex.org/W2128823042","https://openalex.org/W2135995390","https://openalex.org/W2148905283","https://openalex.org/W2171039963","https://openalex.org/W2346318431","https://openalex.org/W2497245348","https://openalex.org/W2511117069","https://openalex.org/W2560140854","https://openalex.org/W2611242648","https://openalex.org/W2751627669","https://openalex.org/W2795596428","https://openalex.org/W2856745662","https://openalex.org/W2884438632","https://openalex.org/W2916904544","https://openalex.org/W2957031687","https://openalex.org/W2983529038","https://openalex.org/W2999030356","https://openalex.org/W3007910004","https://openalex.org/W3013160799","https://openalex.org/W3038925583","https://openalex.org/W3106700156","https://openalex.org/W3155332104","https://openalex.org/W3161516024","https://openalex.org/W3196546806","https://openalex.org/W3203321135","https://openalex.org/W3207598588","https://openalex.org/W4288059402","https://openalex.org/W4292157289","https://openalex.org/W4309674289","https://openalex.org/W4318464200","https://openalex.org/W4366547592","https://openalex.org/W4366549427","https://openalex.org/W4381586920","https://openalex.org/W4384644024","https://openalex.org/W4384662087","https://openalex.org/W4385507096","https://openalex.org/W4385571232","https://openalex.org/W4387031662","https://openalex.org/W4387213334","https://openalex.org/W4387559779","https://openalex.org/W4390897568","https://openalex.org/W4390921168","https://openalex.org/W4391631512","https://openalex.org/W4392341809","https://openalex.org/W4392639864","https://openalex.org/W4394817988","https://openalex.org/W4396822060","https://openalex.org/W4396833223","https://openalex.org/W4398757454","https://openalex.org/W4399248993","https://openalex.org/W4399267535","https://openalex.org/W4399362716","https://openalex.org/W4400222469","https://openalex.org/W4402671506","https://openalex.org/W4403333866","https://openalex.org/W4405627521","https://openalex.org/W4406602486","https://openalex.org/W4409720532","https://openalex.org/W4409749092","https://openalex.org/W4409749383","https://openalex.org/W4411630296","https://openalex.org/W4416852258","https://openalex.org/W7084041499"],"related_works":[],"abstract_inverted_index":{"Open-ended":[0],"writing":[1,17],"assignments":[2],"are":[3],"central":[4],"to":[5],"higher":[6],"education,":[7],"yet":[8],"heterogeneous":[9],"submissions":[10],"and":[11,27,62,79,81,105,115,124],"scale":[12],"make":[13],"evaluation":[14,18],"difficult.":[15],"Automated":[16],"(AWE)":[19],"promises":[20],"speed":[21],"but":[22],"often":[23],"trades":[24],"away":[25],"transparency":[26],"sidelines":[28],"human":[29],"judgment.":[30],"This":[31],"paper":[32],"repositions":[33],"the":[34,127],"AI":[35],"as":[36],"an":[37,106],"on-demand":[38],"collaborator":[39],"that":[40],"can":[41],"provide":[42],"specific,":[43],"targeted":[44],"support.":[45],"In":[46],"a":[47,87,102],"formative":[48],"study,":[49],"we":[50,69,93],"expose":[51],"leverage":[52],"points":[53],"in":[54],"three":[55],"cognitive":[56],"dimensions:":[57],"evidence":[58],"identification,":[59],"comparative":[60],"judgment,":[61],"feedback":[63,84],"composition.":[64],"Guided":[65],"by":[66,132],"these":[67],"insights,":[68],"build":[70],"EvaluAId,":[71],"which":[72],"supports":[73,98],"interactive":[74],"rubric-content":[75],"mapping,":[76],"adaptive":[77],"benchmarking":[78],"self-calibration,":[80],"personalized,":[82],"rubric-aligned":[83],"synthesis.":[85],"Through":[86],"within-subjects":[88],"study":[89],"with":[90,101,112,121],"12":[91],"TAs,":[92,122],"evaluate":[94],"how":[95],"this":[96],"approach":[97],"grading":[99],"compared":[100],"rubric+LLM":[103],"chatbot":[104],"LLM-based":[107],"AWE;":[108],"EvaluAId":[109,133],"improved":[110],"alignment":[111],"expert":[113],"ratings":[114],"increased":[116],"graders\u2019":[117],"satisfaction.":[118],"Finally,":[119],"interviews":[120],"instructors,":[123],"students":[125],"underscored":[126],"value":[128],"of":[129],"thoughtfulness":[130],"supported":[131],"while":[134],"surfacing":[135],"practical":[136],"considerations":[137],"for":[138,146],"integration":[139],"into":[140],"classroom.":[141],"Together,":[142],"our":[143],"results":[144],"argue":[145],"deliberate,":[147],"evidence-first,":[148],"human-in-the-loop":[149],"evaluation.":[150]},"counts_by_year":[],"updated_date":"2026-04-25T08:17:42.794288","created_date":"2026-04-14T00:00:00"}