{"id":"https://openalex.org/W7129013692","doi":"https://doi.org/10.1145/3773966.3778008","title":"Code LLMs Still Fall Short of Top Programmers: Evaluating Algorithmic Code Generation Through Computational Thinking","display_name":"Code LLMs Still Fall Short of Top Programmers: Evaluating Algorithmic Code Generation Through Computational Thinking","publication_year":2026,"publication_date":"2026-02-16","ids":{"openalex":"https://openalex.org/W7129013692","doi":"https://doi.org/10.1145/3773966.3778008"},"language":null,"primary_location":{"id":"doi:10.1145/3773966.3778008","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3773966.3778008","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Nineteenth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3773966.3778008","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085175204","display_name":"Shisong Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Shisong Chen","raw_affiliation_strings":["Shanghai Institute of Artificial Intelligence for Education, East China Normal University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Shanghai Institute of Artificial Intelligence for Education, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Ziyu Zhou","orcid":"https://orcid.org/0009-0002-7419-3098"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziyu Zhou","raw_affiliation_strings":["College of Computer Science and Artificial Intelligence, Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"College of Computer Science and Artificial Intelligence, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108818169","display_name":"Yicong Zhao","orcid":"https://orcid.org/0009-0008-1738-8335"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yicong Zhao","raw_affiliation_strings":["College of Computer Science and Artificial Intelligence, Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"College of Computer Science and Artificial Intelligence, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071511791","display_name":"Chengyi Yang","orcid":"https://orcid.org/0000-0002-7023-7543"},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chengyi Yang","raw_affiliation_strings":["Shanghai Institute of Artificial Intelligence for Education, East China Normal University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Shanghai Institute of Artificial Intelligence for Education, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124247917","display_name":"Zhixu Li","orcid":null},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhixu Li","raw_affiliation_strings":["School of Information, Renmin University of China, Beijing, China and School of Smart Governance, Renmin University of China, Beijing, China"],"affiliations":[{"raw_affiliation_string":"School of Information, Renmin University of China, Beijing, China and School of Smart Governance, Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126099275","display_name":"Yanghua Xiao","orcid":null},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanghua Xiao","raw_affiliation_strings":["Shanghai Key Laboratory of Data Science, School of Computer Science, Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Shanghai Key Laboratory of Data Science, School of Computer Science, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126114928","display_name":"Xin Lin","orcid":null},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Lin","raw_affiliation_strings":["Shanghai Institute of Artificial Intelligence for Education, East China Normal University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Shanghai Institute of Artificial Intelligence for Education, East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100693147","display_name":"Xiaojun Meng","orcid":"https://orcid.org/0000-0002-7809-7012"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaojun Meng","raw_affiliation_strings":["Huawei Noah\u2019s Ark Lab, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Huawei Noah\u2019s Ark Lab, Shanghai, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101182327","display_name":"Jiansheng Wei","orcid":null},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiansheng Wei","raw_affiliation_strings":["Huawei Noah\u2019s Ark Lab, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Huawei Noah\u2019s Ark Lab, Shanghai, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5122387972","display_name":"Kuien Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kuien Liu","raw_affiliation_strings":["Institute of Software, Chinese Academy of Sciences, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Institute of Software, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210128818","https://openalex.org/I19820366"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5085175204"],"corresponding_institution_ids":["https://openalex.org/I66867065"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.32973744,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"79","last_page":"88"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.2827000021934509,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.2827000021934509,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.17910000681877136,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.05290000140666962,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.6682999730110168},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.6144999861717224},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5533000230789185},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4968000054359436},{"id":"https://openalex.org/keywords/interdependence","display_name":"Interdependence","score":0.4936000108718872},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.4625999927520752}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.736299991607666},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.6682999730110168},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6144999861717224},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5533000230789185},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4968000054359436},{"id":"https://openalex.org/C185874996","wikidata":"https://www.wikidata.org/wiki/Q269699","display_name":"Interdependence","level":2,"score":0.4936000108718872},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.4625999927520752},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.35199999809265137},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3427000045776367},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3402000069618225},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.33889999985694885},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.3009999990463257},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.28850001096725464},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2824000120162964},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.2676999866962433},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2667999863624573},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.25600001215934753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3773966.3778008","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3773966.3778008","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Nineteenth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3773966.3778008","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3773966.3778008","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Nineteenth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":2,"referenced_works":["https://openalex.org/W2100029531","https://openalex.org/W4385562549"],"related_works":[],"abstract_inverted_index":{"Evaluating":[0],"the":[1,37,67,92,100,153],"coding":[2],"capabilities":[3],"of":[4,123,155],"models":[5],"through":[6],"algorithmic":[7,55,157],"code":[8,45,56,80,126],"generation":[9,57,127],"is":[10,161],"challenging,":[11],"as":[12,34],"it":[13],"requires":[14],"deep":[15],"problem":[16],"understanding":[17],"and":[18,40,79],"complex":[19],"algorithm":[20,75],"design.":[21],"Current":[22],"benchmarks":[23],"suffer":[24],"from":[25,111],"a":[26,53,85,136],"narrow":[27],"focus":[28],"on":[29],"final":[30,101],"execution":[31],"results":[32],"(such":[33],"pass@k),":[35],"neglecting":[36],"crucial":[38],"reasoning":[39],"problem-solving":[41,95],"processes":[42],"inherent":[43],"in":[44,142,149],"generation.":[46,81],"To":[47],"address":[48],"this":[49],"limitation,":[50],"we":[51,134],"introduce":[52],"multi-phase":[54],"benchmark,":[58],"MUPA,":[59],"structured":[60],"around":[61],"human":[62],"computational":[63],"thinking.":[64],"MUPA":[65],"dissects":[66],"evaluation":[68,122],"into":[69,91],"four":[70],"distinct":[71],"phases:":[72],"example":[73],"understanding,":[74],"selection,":[76],"solution":[77],"description,":[78],"This":[82],"framework":[83],"facilitates":[84],"comprehensive":[86],"assessment":[87],"by":[88],"providing":[89],"insights":[90],"model's":[93],"intermediate":[94],"steps,":[96],"rather":[97],"than":[98],"just":[99],"code.":[102],"We":[103],"manually":[104],"curated":[105],"197":[106],"high-quality":[107],"competitive":[108],"programming":[109],"problems":[110],"Codeforces.":[112],"Utilizing":[113],"an":[114,143],"LLM-as-a-judge":[115],"paradigm":[116],"with":[117],"specialized":[118],"prompts,":[119],"our":[120],"rigorous":[121],"several":[124],"existing":[125],"LLMs":[128],"reveals":[129],"significant":[130],"across-the-board":[131],"challenges.":[132],"Notably,":[133],"establish":[135],"positive":[137],"correlation,":[138],"indicating":[139],"that":[140],"proficiency":[141],"earlier":[144],"phase":[145],"directly":[146],"impacts":[147],"performance":[148],"subsequent":[150],"phases,":[151],"underscoring":[152],"interdependency":[154],"these":[156],"skills.":[158],"The":[159],"benchmark":[160],"publicly":[162],"available":[163],"at":[164],"https://github.com/cheniison/MUPA.":[165]},"counts_by_year":[],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2026-02-17T00:00:00"}