{"id":"https://openalex.org/W4404990707","doi":"https://doi.org/10.1145/3737885","title":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","display_name":"Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers","publication_year":2025,"publication_date":"2025-05-29","ids":{"openalex":"https://openalex.org/W4404990707","doi":"https://doi.org/10.1145/3737885"},"language":"en","primary_location":{"id":"doi:10.1145/3737885","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3737885","pdf_url":null,"source":{"id":"https://openalex.org/S2764465379","display_name":"ACM Transactions on Computing Education","issn_l":"1946-6226","issn":["1946-6226"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computing Education","raw_type":"journal-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2411.18151","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056326360","display_name":"Neil C. C. Brown","orcid":"https://orcid.org/0000-0001-6086-2479"},"institutions":[{"id":"https://openalex.org/I183935753","display_name":"King's College London","ror":"https://ror.org/0220mzb33","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I183935753"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Neil C. C. Brown","raw_affiliation_strings":["King\u2019s College London, London, United Kingdom of Great Britain and Northern Ireland"],"affiliations":[{"raw_affiliation_string":"King\u2019s College London, London, United Kingdom of Great Britain and Northern Ireland","institution_ids":["https://openalex.org/I183935753"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022670725","display_name":"Pierre Weill-Tessier","orcid":"https://orcid.org/0000-0002-6999-8767"},"institutions":[{"id":"https://openalex.org/I183935753","display_name":"King's College London","ror":"https://ror.org/0220mzb33","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I183935753"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Pierre Weill-Tessier","raw_affiliation_strings":["King\u2019s College London, London, United Kingdom of Great Britain and Northern Ireland"],"affiliations":[{"raw_affiliation_string":"King\u2019s College London, London, United Kingdom of Great Britain and Northern Ireland","institution_ids":["https://openalex.org/I183935753"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041367899","display_name":"Juho Leinonen","orcid":"https://orcid.org/0000-0001-6829-9449"},"institutions":[{"id":"https://openalex.org/I9927081","display_name":"Aalto University","ror":"https://ror.org/020hwjq30","country_code":"FI","type":"education","lineage":["https://openalex.org/I9927081"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Juho Leinonen","raw_affiliation_strings":["Aalto University, Aalto, Finland"],"affiliations":[{"raw_affiliation_string":"Aalto University, Aalto, Finland","institution_ids":["https://openalex.org/I9927081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032890999","display_name":"Paul Denny","orcid":"https://orcid.org/0000-0002-5150-9806"},"institutions":[{"id":"https://openalex.org/I154130895","display_name":"University of Auckland","ror":"https://ror.org/03b94tp07","country_code":"NZ","type":"education","lineage":["https://openalex.org/I154130895"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"Paul Denny","raw_affiliation_strings":["The University of Auckland, Auckland, New Zealand"],"affiliations":[{"raw_affiliation_string":"The University of Auckland, Auckland, New Zealand","institution_ids":["https://openalex.org/I154130895"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5018024311","display_name":"Michael K\u00f6lling","orcid":"https://orcid.org/0000-0003-0544-2003"},"institutions":[{"id":"https://openalex.org/I183935753","display_name":"King's College London","ror":"https://ror.org/0220mzb33","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I183935753"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Michael K\u00f6lling","raw_affiliation_strings":["King\u2019s College London, London, United Kingdom of Great Britain and Northern Ireland"],"affiliations":[{"raw_affiliation_string":"King\u2019s College London, London, United Kingdom of Great Britain and Northern Ireland","institution_ids":["https://openalex.org/I183935753"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5056326360"],"corresponding_institution_ids":["https://openalex.org/I183935753"],"apc_list":null,"apc_paid":null,"fwci":4.5896,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.93264784,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"25","issue":"3","first_page":"1","last_page":"43"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9743000268936157,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9743000268936157,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.957099974155426,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9247999787330627,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/judgement","display_name":"Judgement","score":0.886746346950531},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.654310405254364},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4459743797779083},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.09701764583587646}],"concepts":[{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.886746346950531},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.654310405254364},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4459743797779083},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.09701764583587646},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1145/3737885","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3737885","pdf_url":null,"source":{"id":"https://openalex.org/S2764465379","display_name":"ACM Transactions on Computing Education","issn_l":"1946-6226","issn":["1946-6226"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Computing Education","raw_type":"journal-article"},{"id":"pmh:oai:kclpure.kcl.ac.uk:openaire/8da0b706-e684-4324-af66-603feb600fed","is_oa":false,"landing_page_url":"https://kclpure.kcl.ac.uk/portal/en/publications/8da0b706-e684-4324-af66-603feb600fed","pdf_url":null,"source":{"id":"https://openalex.org/S4306400216","display_name":"Research Portal (King's College London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I183935753","host_organization_name":"King's College London","host_organization_lineage":["https://openalex.org/I183935753"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Brown, N C C, Weill-Tessier, P, Leinonen, J, Denny, P & K\u00f6lling, M 2025, 'Howzat? Appealing to Expert Judgement for Evaluating Human and AI Next-Step Hints for Novice Programmers', ACM Transactions on Computing Education (TOCE), vol. 25, no. 3, 31. https://doi.org/10.1145/3737885","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:arXiv.org:2411.18151","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.18151","pdf_url":"https://arxiv.org/pdf/2411.18151","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:share.osf.io:e42bfbae-d663-4314-bf35-e0cd1389dcb7","is_oa":false,"landing_page_url":"https://osf.io/p436s","pdf_url":null,"source":{"id":"https://openalex.org/S4306401127","display_name":"OSF Preprints (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Project"},{"id":"doi:10.48550/arxiv.2411.18151","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.18151","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2411.18151","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.18151","pdf_url":"https://arxiv.org/pdf/2411.18151","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G699743167","display_name":null,"funder_award_id":"356114","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"},{"id":"https://openalex.org/G8616235745","display_name":"Large Language Models for Computing Education","funder_award_id":"356114","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"}],"funders":[{"id":"https://openalex.org/F4320321108","display_name":"Academy of Finland","ror":"https://ror.org/05k73zm37"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404990707.pdf"},"referenced_works_count":89,"referenced_works":["https://openalex.org/W2077361439","https://openalex.org/W3043358777","https://openalex.org/W2237600160","https://openalex.org/W2102636708","https://openalex.org/W2886994154","https://openalex.org/W4403564662","https://openalex.org/W2971797472","https://openalex.org/W2612252595","https://openalex.org/W4387025697","https://openalex.org/W2057833160","https://openalex.org/W2966983573","https://openalex.org/W3010197425","https://openalex.org/W4234580964","https://openalex.org/W2105543722","https://openalex.org/W2783082347","https://openalex.org/W4400267602","https://openalex.org/W4392564669","https://openalex.org/W4398796607","https://openalex.org/W4390985631","https://openalex.org/W3163358064","https://openalex.org/W2138484699","https://openalex.org/W1995942300","https://openalex.org/W4366596891","https://openalex.org/W3045123719","https://openalex.org/W4400210683","https://openalex.org/W2288486118","https://openalex.org/W3046536057","https://openalex.org/W2772911939","https://openalex.org/W2809378240","https://openalex.org/W4380353816","https://openalex.org/W1930624869","https://openalex.org/W2099813784","https://openalex.org/W2808208303","https://openalex.org/W4313288661","https://openalex.org/W4385758784","https://openalex.org/W1607283167","https://openalex.org/W4321162272","https://openalex.org/W4384834951","https://openalex.org/W2894540915","https://openalex.org/W4390621504","https://openalex.org/W6736032515","https://openalex.org/W2106779500","https://openalex.org/W4407681796","https://openalex.org/W4400267611","https://openalex.org/W4386584937","https://openalex.org/W4382654294","https://openalex.org/W4323033692","https://openalex.org/W4391584331","https://openalex.org/W4402788720","https://openalex.org/W4400267620","https://openalex.org/W4323033814","https://openalex.org/W4387015969","https://openalex.org/W4404874258","https://openalex.org/W2966150097","https://openalex.org/W2963562081","https://openalex.org/W4312971677","https://openalex.org/W2970680470","https://openalex.org/W2016060495","https://openalex.org/W4394745423","https://openalex.org/W4392564315","https://openalex.org/W4391417542","https://openalex.org/W3173705641","https://openalex.org/W4393027606","https://openalex.org/W2729767053","https://openalex.org/W1985858262","https://openalex.org/W4402454451","https://openalex.org/W4323037584","https://openalex.org/W2885271276","https://openalex.org/W2747427841","https://openalex.org/W4386099272","https://openalex.org/W4401379837","https://openalex.org/W3172929585","https://openalex.org/W3039376843","https://openalex.org/W4310972872","https://openalex.org/W1972118683","https://openalex.org/W4390490752","https://openalex.org/W2937907378","https://openalex.org/W4407949461","https://openalex.org/W4392564770","https://openalex.org/W4390490804","https://openalex.org/W2135995390","https://openalex.org/W2916390547","https://openalex.org/W2611588359","https://openalex.org/W4361271907","https://openalex.org/W3135482483","https://openalex.org/W2157395790","https://openalex.org/W4393967925","https://openalex.org/W4398785929","https://openalex.org/W4366548330"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2886802431","https://openalex.org/W4388216822","https://openalex.org/W1761762290","https://openalex.org/W4391191813","https://openalex.org/W4387575966","https://openalex.org/W2384262901","https://openalex.org/W2975214487"],"abstract_inverted_index":{"Motivation":[0],":":[1,85,199,296],"Students":[2],"learning":[3],"to":[4,30,53,66,95,121,161,247,281,345,361,364],"program":[5],"often":[6],"reach":[7],"states":[8,109],"where":[9,25],"they":[10,149],"are":[11,359],"stuck":[12],"and":[13,46,64,83,135,268,288,318,343],"can":[14,40],"make":[15,43],"no":[16,26,157,291],"forward":[17,44],"progress\u2014but":[18],"this":[19,33,351],"may":[20],"be":[21,362,374],"outside":[22],"the":[23,93,143,162,166,182,241,253,283,311,366],"classroom":[24],"instructor":[27],"is":[28,51,302,314],"available":[29],"help.":[31],"In":[32],"situation,":[34],"an":[35,98,377],"automatically":[36,70],"generated":[37,132,155],"next-step":[38,127,211],"hint":[39,59,191,261],"help":[41],"them":[42],"progress":[45],"support":[47],"their":[48,357],"learning.":[49],"It":[50],"important":[52,256,323],"know":[54],"what":[55,165],"makes":[56],"a":[57,61,103,115,123,174,187,194,225,248,259,335,347],"good":[58,68,260],"or":[60,274],"bad":[62],"hint,":[63],"how":[65,148],"generate":[67],"hints":[69,144,153,183,212,301],"in":[71,97,173,207,232,376],"novice":[72],"programming":[73,214],"tools,":[74],"for":[75,213],"example,":[76],"using":[77,193],"large":[78],"language":[79],"models":[80,220],"(LLMs).":[81],"Method":[82],"Participants":[84,113,141,170],"We":[86,101,200],"recruited":[87],"44":[88],"Java":[89,128],"educators":[90],"from":[91],"around":[92],"world":[94],"participate":[96],"online":[99],"study.":[100],"used":[102,114,223],"set":[104,124,188],"of":[105,125,181,189,258,293,299,328,337,341],"real":[106],"student":[107],"code":[108],"as":[110,118],"hint-generation":[111],"scenarios.":[112],"technique":[116],"known":[117],"comparative":[119],"judgement":[120],"rank":[122],"candidate":[126],"hints,":[129],"which":[130],"were":[131,150,154,184,262,321],"by":[133,136],"LLMs":[134,203,307],"five":[137],"human":[138,230],"experienced":[139],"educators.":[140],"ranked":[142],"without":[145],"being":[146,266,276],"told":[147],"generated.":[151],"The":[152,179,330,369],"with":[156,176,216,224],"explicit":[158],"detail":[159],"given":[160,305],"LLMs/humans":[163],"on":[164,356],"target":[167],"task":[168,313],"was.":[169],"then":[171],"filled":[172],"survey":[175],"follow-up":[177],"questions.":[178],"ranks":[180],"analysed":[185],"against":[186],"extracted":[190],"characteristics":[192],"random":[195,250],"forest":[196,251],"approach.":[197],"Findings":[198],"found":[201,290],"that":[202,306,332,349,354],"had":[204],"considerable":[205],"variation":[206],"generating":[208,233],"high":[209],"quality":[210],"novices,":[215],"GPT-4":[217,228],"outperforming":[218],"other":[219],"tested.":[221],"When":[222],"well-designed":[226],"prompt,":[227],"outperformed":[229,308],"experts":[231,338],"pedagogically":[234],"valuable":[235],"hints.":[236,329],"A":[237],"multi-stage":[238],"prompt":[239,348],"was":[240,285],"most":[242,255],"effective":[243],"LLM":[244],"prompt.":[245],"According":[246],"fitted":[249],"model,":[252],"two":[254],"factors":[257],"length":[263,317],"(80\u2013160":[264],"words":[265],"best)":[267],"reading":[269,319],"level":[270,320],"(US":[271],"grade":[272],"nine":[273],"below":[275],"best).":[277],"Offering":[278],"alternative":[279],"approaches":[280],"solving":[282],"problem":[284],"considered":[286],"bad,":[287],"we":[289],"effect":[292],"sentiment.":[294],"Conclusions":[295],"Automatic":[297],"generation":[298],"these":[300],"immediately":[303],"viable,":[304],"humans\u2014even":[309],"when":[310],"students\u2019":[312],"unknown.":[315],"Hint":[316],"more":[322],"than":[324],"several":[325,339],"pedagogical":[326],"features":[327],"fact":[331],"it":[333],"took":[334],"group":[336],"rounds":[340],"experimentation":[342],"refinement":[344],"design":[346],"achieves":[350],"outcome":[352],"suggests":[353],"students":[355],"own":[358],"unlikely":[360],"able":[363],"produce":[365],"same":[367],"benefit.":[368],"prompting":[370],"task,":[371],"therefore,":[372],"should":[373],"embedded":[375],"expert-designed":[378],"tool.":[379]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2024-12-05T00:00:00"}