{"id":"https://openalex.org/W4403564662","doi":"https://doi.org/10.1145/3699538.3699556","title":"One Step at a Time: Combining LLMs and Static Analysis to Generate Next-Step Hints for Programming Tasks","display_name":"One Step at a Time: Combining LLMs and Static Analysis to Generate Next-Step Hints for Programming Tasks","publication_year":2024,"publication_date":"2024-11-12","ids":{"openalex":"https://openalex.org/W4403564662","doi":"https://doi.org/10.1145/3699538.3699556"},"language":"en","primary_location":{"id":"doi:10.1145/3699538.3699556","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3699538.3699556","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.09268","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074151977","display_name":"Anastasiia Birillo","orcid":"https://orcid.org/0000-0003-2269-8211"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Anastasiia Birillo","raw_affiliation_strings":["JetBrains Research, Belgrade, Serbia"],"raw_orcid":"https://orcid.org/0000-0003-2269-8211","affiliations":[{"raw_affiliation_string":"JetBrains Research, Belgrade, Serbia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015564606","display_name":"Elizaveta Artser","orcid":"https://orcid.org/0009-0007-4919-2698"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Elizaveta Artser","raw_affiliation_strings":["JetBrains Research, Munich, Germany"],"raw_orcid":"https://orcid.org/0009-0007-4919-2698","affiliations":[{"raw_affiliation_string":"JetBrains Research, Munich, Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093800119","display_name":"Anna Potriasaeva","orcid":"https://orcid.org/0009-0004-4686-2412"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Anna Potriasaeva","raw_affiliation_strings":["JetBrains Research, Belgrade, Serbia"],"raw_orcid":"https://orcid.org/0009-0004-4686-2412","affiliations":[{"raw_affiliation_string":"JetBrains Research, Belgrade, Serbia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077202915","display_name":"Ilya Vlasov","orcid":"https://orcid.org/0000-0002-2744-8345"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ilya Vlasov","raw_affiliation_strings":["JetBrains Research, Belgrade, Serbia"],"raw_orcid":"https://orcid.org/0000-0002-2744-8345","affiliations":[{"raw_affiliation_string":"JetBrains Research, Belgrade, Serbia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106342298","display_name":"Katsiaryna Dzialets","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Katsiaryna Dzialets","raw_affiliation_strings":["JetBrains, Munich, Germany"],"raw_orcid":"https://orcid.org/0009-0003-3269-3622","affiliations":[{"raw_affiliation_string":"JetBrains, Munich, Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013316832","display_name":"Yaroslav Golubev","orcid":"https://orcid.org/0000-0001-7009-635X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yaroslav Golubev","raw_affiliation_strings":["JetBrains Research, Belgrade, Serbia"],"raw_orcid":"https://orcid.org/0000-0001-7009-635X","affiliations":[{"raw_affiliation_string":"JetBrains Research, Belgrade, Serbia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102373030","display_name":"Igor Gerasimov","orcid":"https://orcid.org/0009-0006-1887-5317"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Igor Gerasimov","raw_affiliation_strings":["JetBrains, Berlin, Germany"],"raw_orcid":"https://orcid.org/0009-0006-1887-5317","affiliations":[{"raw_affiliation_string":"JetBrains, Berlin, Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052659996","display_name":"Hieke Keuning","orcid":"https://orcid.org/0000-0001-5778-7519"},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Hieke Keuning","raw_affiliation_strings":["Utrecht University, Utrecht, Netherlands"],"raw_orcid":"https://orcid.org/0000-0001-5778-7519","affiliations":[{"raw_affiliation_string":"Utrecht University, Utrecht, Netherlands","institution_ids":["https://openalex.org/I193662353"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5065316896","display_name":"Timofey Bryksin","orcid":"https://orcid.org/0000-0001-9022-3563"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Timofey Bryksin","raw_affiliation_strings":["JetBrains Research, Limassol, Cyprus"],"raw_orcid":"https://orcid.org/0000-0001-9022-3563","affiliations":[{"raw_affiliation_string":"JetBrains Research, Limassol, Cyprus","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5074151977"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.2475,"has_fulltext":true,"cited_by_count":16,"citation_normalized_percentile":{"value":0.96767855,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"12"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11122","display_name":"Online Learning and Analytics","score":0.9869999885559082,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8319627046585083},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6887775659561157},{"id":"https://openalex.org/keywords/plug-in","display_name":"Plug-in","score":0.6758285164833069},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5949302911758423},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5779511332511902},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5478271842002869},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.45927056670188904},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4318188726902008},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.333936870098114},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.10530436038970947}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8319627046585083},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6887775659561157},{"id":"https://openalex.org/C4924752","wikidata":"https://www.wikidata.org/wiki/Q184148","display_name":"Plug-in","level":2,"score":0.6758285164833069},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5949302911758423},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5779511332511902},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5478271842002869},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.45927056670188904},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4318188726902008},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.333936870098114},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.10530436038970947},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3699538.3699556","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3699538.3699556","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th Koli Calling International Conference on Computing Education Research","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2410.09268","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.09268","pdf_url":"https://arxiv.org/pdf/2410.09268","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:dspace.library.uu.nl:1874/482513","is_oa":true,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/482513","pdf_url":null,"source":{"id":"https://openalex.org/S4306401649","display_name":"Utrecht University Repository (Utrecht University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193662353","host_organization_name":"Utrecht University","host_organization_lineage":["https://openalex.org/I193662353"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Part of book"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.09268","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.09268","pdf_url":"https://arxiv.org/pdf/2410.09268","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.75,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403564662.pdf","grobid_xml":"https://content.openalex.org/works/W4403564662.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W1976220211","https://openalex.org/W2128715468","https://openalex.org/W2344752727","https://openalex.org/W2604398661","https://openalex.org/W2728220450","https://openalex.org/W2894540915","https://openalex.org/W2962829286","https://openalex.org/W2964148749","https://openalex.org/W3089554664","https://openalex.org/W3113108648","https://openalex.org/W3198599617","https://openalex.org/W4288076218","https://openalex.org/W4313894684","https://openalex.org/W4321120957","https://openalex.org/W4388691793","https://openalex.org/W4390315357","https://openalex.org/W4390490752","https://openalex.org/W4390621504","https://openalex.org/W4391584331","https://openalex.org/W4392563703","https://openalex.org/W4393967925","https://openalex.org/W4394929444","https://openalex.org/W4395029328","https://openalex.org/W4396833177","https://openalex.org/W4400833413","https://openalex.org/W4401391312"],"related_works":["https://openalex.org/W47352601","https://openalex.org/W2981957539","https://openalex.org/W4287378204","https://openalex.org/W2461489085","https://openalex.org/W4286950177","https://openalex.org/W4285337522","https://openalex.org/W3203498468","https://openalex.org/W2464866501","https://openalex.org/W2289852464","https://openalex.org/W2788261136"],"abstract_inverted_index":{"Students":[0],"often":[1],"struggle":[2],"with":[3,19,71,96,158,254,287],"solving":[4],"programming":[5,129],"problems":[6],"when":[7,12],"learning":[8],"to":[9,15,54,57,59,67,121,156,170,180,193,197,210,285],"code,":[10],"especially":[11],"they":[13,52],"have":[14],"do":[16,55],"it":[17],"online,":[18],"one":[20],"of":[21,26,32,133,144,154,230,242,266],"the":[22,30,60,78,106,134,159,162,168,172,178,186,194,211,248,267,277,280,288],"most":[23,79],"common":[24],"disadvantages":[25],"working":[27],"online":[28],"being":[29,76],"lack":[31],"personalized":[33,92],"help.":[34],"This":[35],"help":[36],"can":[37,103],"be":[38],"provided":[39],"as":[40,100,207],"next-step":[41,249],"hint":[42],"generation,":[43],"i.e.,":[44],"showing":[45],"a":[46,87,118,138,152,208,228,252],"student":[47],"what":[48],"specific":[49],"small":[50],"step":[51],"need":[53],"next":[56,173],"get":[58],"correct":[61],"solution.":[62],"There":[63],"are":[64],"many":[65],"ways":[66],"generate":[68],"such":[69,99],"hints,":[70],"large":[72],"language":[73],"models":[74],"(LLMs)":[75],"among":[77],"actively":[80],"studied":[81],"right":[82],"now.":[83],"While":[84],"LLMs":[85],"constitute":[86],"promising":[88],"technology":[89],"for":[90,128,232,276],"providing":[91],"help,":[93],"combining":[94],"them":[95,284],"other":[97],"techniques,":[98],"static":[101,191],"analysis,":[102],"significantly":[104],"improve":[105],"output":[107],"quality.":[108,202],"In":[109],"this":[110,114,182],"work,":[111],"we":[112,189,226,246],"utilize":[113],"idea":[115],"and":[116,125,142,175,201,238,271,279],"propose":[117,227],"novel":[119],"system":[120,282],"provide":[122],"both":[123,264],"textual":[124,270],"code":[126,169,196,272],"hints":[127,250,268],"tasks.":[130,290],"The":[131,203],"pipeline":[132,237],"proposed":[135,281],"approach":[136],"uses":[137],"chain-of-thought":[139],"prompting":[140],"technique":[141],"consists":[143],"three":[145],"distinct":[146],"steps:":[147],"(1)":[148],"generating":[149,167,177],"subgoals":[150],"-":[151,269,273],"list":[153,229],"actions":[155],"proceed":[157,286],"task":[160],"from":[161,257],"current":[163],"student's":[164],"solution,":[165],"(2)":[166],"achieve":[171],"subgoal,":[174],"(3)":[176],"text":[179],"describe":[181],"needed":[183],"action.":[184],"During":[185],"second":[187],"step,":[188],"apply":[190],"analysis":[192],"generated":[195],"control":[198],"its":[199],"size":[200],"tool":[204],"is":[205],"implemented":[206],"modification":[209],"open-source":[212],"JetBrains":[213],"Academy":[214],"plugin,":[215],"supporting":[216],"students":[217,256],"in":[218,235,251],"their":[219],"in-IDE":[220],"courses.":[221],"To":[222],"evaluate":[223,247],"our":[224,236],"approach,":[225],"criteria":[231],"all":[233],"steps":[234],"conduct":[239],"two":[240,258],"rounds":[241],"expert":[243],"validation.":[244],"Finally,":[245],"classroom":[253],"14":[255],"universities.":[259],"Our":[260],"results":[261],"show":[262],"that":[263],"forms":[265],"were":[274],"helpful":[275],"students,":[278],"helped":[283],"coding":[289]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":14}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}