{"id":"https://openalex.org/W7155649610","doi":"https://doi.org/10.1145/3785022.3785130","title":"When LLMs Fall Short in Deductive Coding: Model Comparisons and Human\u2013AI Collaboration Workflow Design","display_name":"When LLMs Fall Short in Deductive Coding: Model Comparisons and Human\u2013AI Collaboration Workflow Design","publication_year":2026,"publication_date":"2026-04-25","ids":{"openalex":"https://openalex.org/W7155649610","doi":"https://doi.org/10.1145/3785022.3785130"},"language":null,"primary_location":{"id":"doi:10.1145/3785022.3785130","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3785022.3785130","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the LAK26: 16th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3785022.3785130","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100406500","display_name":"Zijian Li","orcid":"https://orcid.org/0000-0002-8057-2257"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Zijian Li","raw_affiliation_strings":["Department of Educational Technology, Graduate School of Education, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0000-0566-9051","affiliations":[{"raw_affiliation_string":"Department of Educational Technology, Graduate School of Education, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114190756","display_name":"Luzhen Tang","orcid":null},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Luzhen Tang","raw_affiliation_strings":["Department of Educational Technology, Graduate School of Education, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0005-0102-5677","affiliations":[{"raw_affiliation_string":"Department of Educational Technology, Graduate School of Education, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128672851","display_name":"Mengyu Xia","orcid":null},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mengyu Xia","raw_affiliation_strings":["Department of Educational Technology, Graduate School of Education, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0001-8496-8611","affiliations":[{"raw_affiliation_string":"Department of Educational Technology, Graduate School of Education, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100406123","display_name":"Xinyu Li","orcid":"https://orcid.org/0000-0003-2681-4451"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Xinyu Li","raw_affiliation_strings":["Centre for Learning Analytics, Faculty of Information Technology, Monash University, Melbourne, Australia"],"raw_orcid":"https://orcid.org/0000-0003-2681-4451","affiliations":[{"raw_affiliation_string":"Centre for Learning Analytics, Faculty of Information Technology, Monash University, Melbourne, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087573557","display_name":"Naping Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I32574673","display_name":"Shantou University","ror":"https://ror.org/01a099706","country_code":"CN","type":"education","lineage":["https://openalex.org/I32574673"]},{"id":"https://openalex.org/I4210120522","display_name":"Shantou University Medical College","ror":"https://ror.org/02gxych78","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210120522"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Naping Chen","raw_affiliation_strings":["Medical College, Shantou University, Shantou, Guangdong, China"],"raw_orcid":"https://orcid.org/0000-0002-3996-5454","affiliations":[{"raw_affiliation_string":"Medical College, Shantou University, Shantou, Guangdong, China","institution_ids":["https://openalex.org/I32574673","https://openalex.org/I4210120522"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036855560","display_name":"Dragan Ga\u0161evi\u0107","orcid":"https://orcid.org/0000-0001-9265-1908"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Dragan Ga\u0161evi\u0107","raw_affiliation_strings":["Centre for Learning Analytics, Faculty of Information Technology, Monash University, Melbourne, Australia"],"raw_orcid":"https://orcid.org/0000-0001-9265-1908","affiliations":[{"raw_affiliation_string":"Centre for Learning Analytics, Faculty of Information Technology, Monash University, Melbourne, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5006305255","display_name":"Yizhou Fan","orcid":"https://orcid.org/0000-0003-2777-1705"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Yizhou Fan","raw_affiliation_strings":["Department of Educational Technology, Graduate School of Education, Peking University, Beijing, China and Centre for Learning Analytics, Faculty of Information Technology, Monash University, Melbourne, Australia"],"raw_orcid":"https://orcid.org/0000-0003-2777-1705","affiliations":[{"raw_affiliation_string":"Department of Educational Technology, Graduate School of Education, Peking University, Beijing, China and Centre for Learning Analytics, Faculty of Information Technology, Monash University, Melbourne, Australia","institution_ids":["https://openalex.org/I56590836"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5100406500"],"corresponding_institution_ids":["https://openalex.org/I20231570"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.95879068,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"685","last_page":"696"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.2892000079154968,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.2892000079154968,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.09650000184774399,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09380000084638596,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.7576000094413757},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.6776999831199646},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.3497999906539917},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.3070000112056732},{"id":"https://openalex.org/keywords/comprehension","display_name":"Comprehension","score":0.26669999957084656}],"concepts":[{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.7576000094413757},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.6776999831199646},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5436000227928162},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.42669999599456787},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3939000070095062},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3546999990940094},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.3497999906539917},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.31940001249313354},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3070000112056732},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.26669999957084656},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.25859999656677246},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.2540000081062317}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3785022.3785130","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3785022.3785130","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the LAK26: 16th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3785022.3785130","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3785022.3785130","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the LAK26: 16th International Learning Analytics and Knowledge Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Decent work and economic growth","score":0.5242528319358826,"id":"https://metadata.un.org/sdg/8"}],"awards":[{"id":"https://openalex.org/G7458305285","display_name":null,"funder_award_id":"62407001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":47,"referenced_works":["https://openalex.org/W1568955415","https://openalex.org/W2016874080","https://openalex.org/W2741271950","https://openalex.org/W2792572858","https://openalex.org/W2971937231","https://openalex.org/W2979647235","https://openalex.org/W3012126397","https://openalex.org/W3210673220","https://openalex.org/W4220717796","https://openalex.org/W4225279564","https://openalex.org/W4289792687","https://openalex.org/W4295700858","https://openalex.org/W4312804966","https://openalex.org/W4316469706","https://openalex.org/W4360978668","https://openalex.org/W4366548345","https://openalex.org/W4378232119","https://openalex.org/W4381144965","https://openalex.org/W4381956531","https://openalex.org/W4385627138","https://openalex.org/W4387851331","https://openalex.org/W4388552174","https://openalex.org/W4389241245","https://openalex.org/W4392445427","https://openalex.org/W4393257072","https://openalex.org/W4396773505","https://openalex.org/W4396833424","https://openalex.org/W4401043286","https://openalex.org/W4401980941","https://openalex.org/W4402163574","https://openalex.org/W4403018474","https://openalex.org/W4403056940","https://openalex.org/W4403637324","https://openalex.org/W4403970100","https://openalex.org/W4405129239","https://openalex.org/W4407841953","https://openalex.org/W4408049766","https://openalex.org/W4408504572","https://openalex.org/W4409166975","https://openalex.org/W4409628585","https://openalex.org/W4409736133","https://openalex.org/W4410299035","https://openalex.org/W4410554739","https://openalex.org/W4412873992","https://openalex.org/W4413072640","https://openalex.org/W4416034330","https://openalex.org/W7131824921"],"related_works":[],"abstract_inverted_index":{"With":[0],"generative":[1],"artificial":[2],"intelligence":[3],"driving":[4],"the":[5,27,31,66,97,161,177,185],"growth":[6],"of":[7,33,68,100,163,180,188],"dialogic":[8],"data":[9],"in":[10,78,108,118,137],"education,":[11],"automated":[12,42],"coding":[13,43,57,91,98,139,151,155],"is":[14],"a":[15,145],"promising":[16],"direction":[17],"for":[18,192],"learning":[19],"analytics":[20],"to":[21,29,46,52,73,114],"improve":[22],"efficiency.":[23],"This":[24],"surge":[25],"highlights":[26],"need":[28],"understand":[30],"nuances":[32],"student-AI":[34],"interactions,":[35],"especially":[36,166],"those":[37],"rare":[38,49],"yet":[39],"crucial.":[40],"However,":[41],"may":[44],"struggle":[45],"capture":[47],"these":[48],"codes":[50],"due":[51],"imbalanced":[53,115],"data,":[54],"while":[55,82,153,183],"human":[56,181],"remains":[58],"time-consuming":[59],"and":[60,106,131,135,143,172,176],"labour-intensive.":[61],"The":[62],"current":[63],"study":[64],"examined":[65],"potential":[67],"large":[69],"language":[70],"models":[71,130],"(LLMs)":[72],"approximate":[74],"or":[75],"replace":[76],"humans":[77],"deductive,":[79],"theory-driven":[80],"coding,":[81],"also":[83],"exploring":[84],"how":[85],"human\u2013AI":[86,146,189],"collaboration":[87],"might":[88],"support":[89],"such":[90],"tasks":[92],"at":[93],"scale.":[94],"We":[95,141],"compared":[96],"performance":[99],"small":[101],"transformer":[102],"classifiers":[103],"(e.g.,":[104],"BERT)":[105],"LLMs":[107,125,164],"two":[109],"datasets,":[110],"with":[111,169],"particular":[112],"attention":[113],"head\u2013tail":[116],"distributions":[117],"dialogue":[119],"codes.":[120],"Our":[121,157],"results":[122],"showed":[123],"that":[124,149],"did":[126],"not":[127],"outperform":[128],"BERT-based":[129],"exhibited":[132],"systematic":[133],"errors":[134],"biases":[136],"deductive":[138],"tasks.":[140],"designed":[142],"evaluated":[144],"collaborative":[147,190],"workflow":[148],"improved":[150],"efficiency":[152],"maintaining":[154],"reliability.":[156],"findings":[158],"reveal":[159],"both":[160],"limitations":[162],"\u2013":[165,175],"their":[167],"difficulties":[168],"semantic":[170],"similarity":[171],"theoretical":[173],"interpretations":[174],"indispensable":[178],"role":[179],"judgment,":[182],"demonstrating":[184],"practical":[186],"promise":[187],"workflows":[191],"coding.":[193]},"counts_by_year":[],"updated_date":"2026-04-29T09:16:38.111599","created_date":"2026-04-26T00:00:00"}