{"id":"https://openalex.org/W7128789123","doi":"https://doi.org/10.1145/3770762.3772536","title":"Deriving Instructional Insights from Human-LLM Co-Evaluation of Student Collaboration in Data-Centric Programming","display_name":"Deriving Instructional Insights from Human-LLM Co-Evaluation of Student Collaboration in Data-Centric Programming","publication_year":2026,"publication_date":"2026-02-13","ids":{"openalex":"https://openalex.org/W7128789123","doi":"https://doi.org/10.1145/3770762.3772536"},"language":null,"primary_location":{"id":"doi:10.1145/3770762.3772536","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3770762.3772536","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.1","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3770762.3772536","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076326571","display_name":"Marshall An","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Marshall An","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA"],"raw_orcid":"https://orcid.org/0009-0005-5165-640X","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103161572","display_name":"Christine Kwon","orcid":"https://orcid.org/0009-0001-2825-2280"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christine Kwon","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA"],"raw_orcid":"https://orcid.org/0009-0001-2825-2280","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yoonjae Lee","orcid":"https://orcid.org/0000-0002-3762-0639"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Yoonjae Lee","raw_affiliation_strings":["Seoul National University, Seoul, Republic of Korea"],"raw_orcid":"https://orcid.org/0000-0002-3762-0639","affiliations":[{"raw_affiliation_string":"Seoul National University, Seoul, Republic of Korea","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045662300","display_name":"Ji-Hyeon Hur","orcid":"https://orcid.org/0000-0003-4065-1476"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Ji-Hyeon Hur","raw_affiliation_strings":["Seoul National University, Seoul, Republic of Korea"],"raw_orcid":"https://orcid.org/0000-0003-4065-1476","affiliations":[{"raw_affiliation_string":"Seoul National University, Seoul, Republic of Korea","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Dongho Lee","orcid":"https://orcid.org/0009-0008-5941-9208"},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Dongho Lee","raw_affiliation_strings":["Seoul National University, Seoul, Republic of Korea"],"raw_orcid":"https://orcid.org/0009-0008-5941-9208","affiliations":[{"raw_affiliation_string":"Seoul National University, Seoul, Republic of Korea","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125960446","display_name":"Vincent Huai","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vincent Huai","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA"],"raw_orcid":"https://orcid.org/0009-0005-7909-5421","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125906295","display_name":"Barry Zheng","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Barry Zheng","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA"],"raw_orcid":"https://orcid.org/0009-0002-1538-8186","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125901232","display_name":"Matthew Yu","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Yu","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA"],"raw_orcid":"https://orcid.org/0009-0007-6064-2418","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Joana Liu","orcid":"https://orcid.org/0009-0005-6077-2998"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joana Liu","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA"],"raw_orcid":"https://orcid.org/0009-0005-6077-2998","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111824625","display_name":"Jenny Pugh","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jenny Pugh","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA"],"raw_orcid":"https://orcid.org/0009-0007-6259-8502","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125964759","display_name":"Gahgene Gweon","orcid":null},"institutions":[{"id":"https://openalex.org/I139264467","display_name":"Seoul National University","ror":"https://ror.org/04h9pn542","country_code":"KR","type":"education","lineage":["https://openalex.org/I139264467"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Gahgene Gweon","raw_affiliation_strings":["Seoul National University, Seoul, Republic of Korea"],"raw_orcid":"https://orcid.org/0000-0003-3268-477X","affiliations":[{"raw_affiliation_string":"Seoul National University, Seoul, Republic of Korea","institution_ids":["https://openalex.org/I139264467"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5122985821","display_name":"John Stamper","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Stamper","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA"],"raw_orcid":"https://orcid.org/0000-0002-2291-1468","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"45","last_page":"51"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10636","display_name":"Innovative Teaching and Learning Methods","score":0.5266000032424927,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10636","display_name":"Innovative Teaching and Learning Methods","score":0.5266000032424927,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.07240000367164612,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13937","display_name":"Genetics, Bioinformatics, and Biomedical Research","score":0.03680000081658363,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6978999972343445},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.6108999848365784},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5376999974250793},{"id":"https://openalex.org/keywords/negotiation","display_name":"Negotiation","score":0.4875999987125397},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4546999931335449},{"id":"https://openalex.org/keywords/collaborative-design","display_name":"Collaborative design","score":0.44929999113082886},{"id":"https://openalex.org/keywords/computer-supported-collaborative-learning","display_name":"Computer-supported collaborative learning","score":0.4481000006198883},{"id":"https://openalex.org/keywords/replicate","display_name":"Replicate","score":0.43309998512268066},{"id":"https://openalex.org/keywords/protocol-analysis","display_name":"Protocol analysis","score":0.4147999882698059}],"concepts":[{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6978999972343445},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6317999958992004},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.6108999848365784},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5376999974250793},{"id":"https://openalex.org/C199776023","wikidata":"https://www.wikidata.org/wiki/Q202875","display_name":"Negotiation","level":2,"score":0.4875999987125397},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4546999931335449},{"id":"https://openalex.org/C2982907496","wikidata":"https://www.wikidata.org/wiki/Q1145523","display_name":"Collaborative design","level":3,"score":0.44929999113082886},{"id":"https://openalex.org/C2778515922","wikidata":"https://www.wikidata.org/wiki/Q1640981","display_name":"Computer-supported collaborative learning","level":3,"score":0.4481000006198883},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.43309998512268066},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.4242999851703644},{"id":"https://openalex.org/C133112747","wikidata":"https://www.wikidata.org/wiki/Q7251931","display_name":"Protocol analysis","level":2,"score":0.4147999882698059},{"id":"https://openalex.org/C85973986","wikidata":"https://www.wikidata.org/wiki/Q1091731","display_name":"Exploratory research","level":2,"score":0.40860000252723694},{"id":"https://openalex.org/C87156501","wikidata":"https://www.wikidata.org/wiki/Q7268708","display_name":"Qualitative property","level":2,"score":0.4032000005245209},{"id":"https://openalex.org/C190248442","wikidata":"https://www.wikidata.org/wiki/Q839486","display_name":"Qualitative research","level":2,"score":0.4025999903678894},{"id":"https://openalex.org/C138020889","wikidata":"https://www.wikidata.org/wiki/Q2349659","display_name":"Collaborative learning","level":2,"score":0.3869999945163727},{"id":"https://openalex.org/C159456220","wikidata":"https://www.wikidata.org/wiki/Q667334","display_name":"Instructional design","level":2,"score":0.3562999963760376},{"id":"https://openalex.org/C51672120","wikidata":"https://www.wikidata.org/wiki/Q303446","display_name":"Cooperative learning","level":3,"score":0.34950000047683716},{"id":"https://openalex.org/C3018587665","wikidata":"https://www.wikidata.org/wiki/Q7268696","display_name":"Qualitative analysis","level":3,"score":0.3490999937057495},{"id":"https://openalex.org/C554579003","wikidata":"https://www.wikidata.org/wiki/Q474157","display_name":"Collaborative software","level":2,"score":0.32670000195503235},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3237999975681305},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.31630000472068787},{"id":"https://openalex.org/C111226992","wikidata":"https://www.wikidata.org/wiki/Q626225","display_name":"Teamwork","level":2,"score":0.30809998512268066},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.30250000953674316},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.27379998564720154},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.26989999413490295},{"id":"https://openalex.org/C109778625","wikidata":"https://www.wikidata.org/wiki/Q796573","display_name":"Computer-mediated communication","level":3,"score":0.2563000023365021}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3770762.3772536","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3770762.3772536","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.1","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3770762.3772536","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3770762.3772536","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th ACM Technical Symposium on Computer Science Education V.1","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.5257784724235535,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W129847848","https://openalex.org/W1529223212","https://openalex.org/W1551289443","https://openalex.org/W2039019646","https://openalex.org/W2065443038","https://openalex.org/W2087725383","https://openalex.org/W2114293844","https://openalex.org/W2128677857","https://openalex.org/W2744903847","https://openalex.org/W2800397035","https://openalex.org/W2969653701","https://openalex.org/W3037795665","https://openalex.org/W3119305695","https://openalex.org/W4205737716","https://openalex.org/W4392445412","https://openalex.org/W4392484144","https://openalex.org/W4392484188","https://openalex.org/W4396831993","https://openalex.org/W4399103031","https://openalex.org/W4401042136","https://openalex.org/W4403220991","https://openalex.org/W4404783774","https://openalex.org/W4404792888","https://openalex.org/W4411269302","https://openalex.org/W4411272222"],"related_works":[],"abstract_inverted_index":{"This":[0],"quasi-experimental":[1],"study":[2],"integrates":[3],"a":[4,44,79,87,151],"large":[5],"language":[6],"model":[7],"(LLM)":[8],"with":[9,47,61],"expert":[10],"qualitative":[11,102],"analysis":[12],"to":[13,164,171],"examine":[14],"how":[15,111,135],"instructional":[16,128],"design":[17,77],"variations":[18],"in":[19,26,71],"computer-supported":[20],"collaborative":[21,141],"learning":[22],"(CSCL)":[23],"shape":[24],"collaboration":[25],"data-centric":[27],"programming.":[28],"We":[29],"collected":[30],"73":[31],"team":[32],"transcripts":[33],"from":[34],"two":[35],"contrasting":[36],"CSCL":[37,137],"designs":[38,138],"deployed":[39],"across":[40],"five":[41],"course":[42],"offerings:":[43],"closed-ended":[45,119],"variant":[46,57],"prescribed":[48],"solution":[49,116],"paths":[50],"and":[51,54,92,106,149,168],"auto-graded":[52],"milestones,":[53],"an":[55],"open-ended":[56,76,112,136],"supporting":[58],"exploratory":[59],"tasks":[60,113],"multiple":[62],"valid":[63],"paths.":[64],"LLM":[65],"annotation":[66],"revealed":[67],"statistically":[68],"significant":[69],"differences":[70],"knowledge":[72],"co-construction":[73],"patterns:":[74],"the":[75,158],"yielded":[78],"higher":[80],"proportion":[81],"of":[82,90],"utterances":[83],"focused":[84],"on":[85],"developing":[86],"shared":[88],"understanding":[89],"problems":[91],"solutions.":[93],"Guided":[94],"by":[95],"these":[96,108],"quantitative":[97],"results,":[98],"human":[99],"experts":[100],"conducted":[101],"coding":[103],"that":[104],"confirmed":[105],"enriched":[107],"findings,":[109],"showing":[110],"fostered":[114],"elaborative":[115],"negotiation":[117],"while":[118],"structures":[120],"promoted":[121],"non-elaborative":[122],"exchanges.":[123],"Our":[124],"contributions":[125],"are:":[126],"(1)":[127],"insights":[129],"for":[130,144,154,162],"data":[131,146],"science":[132,147],"education,":[133],"demonstrating":[134],"better":[139],"support":[140],"sense-making":[142],"essential":[143],"real-world":[145],"projects;":[148],"(2)":[150],"documented":[152],"workflow":[153],"human-LLM":[155],"co-evaluation,":[156],"providing":[157],"methodological":[159],"detail":[160],"necessary":[161],"others":[163],"replicate":[165],"our":[166],"process":[167],"apply":[169],"it":[170],"future":[172],"studies.":[173]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-02-14T00:00:00"}