{"id":"https://openalex.org/W7220455017","doi":"https://doi.org/10.48550/arxiv.2610.02438","title":"Are you Synthesizing or Recalling? Evaluating LLMs on Algorithmic Code Retrieval","display_name":"Are you Synthesizing or Recalling? Evaluating LLMs on Algorithmic Code Retrieval","publication_year":2026,"publication_date":"2026-10-01","ids":{"openalex":"https://openalex.org/W7220455017","doi":"https://doi.org/10.48550/arxiv.2610.02438"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.02438","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.02438","pdf_url":"https://arxiv.org/pdf/2610.02438","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.02438","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041034756","display_name":"Nickil Maveli","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Maveli, Nickil","raw_affiliation_strings":["School of Informatics , University of Edinburgh 10 Crichton Street , Edinburgh , EH8 9AB"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Informatics , University of Edinburgh 10 Crichton Street , Edinburgh , EH8 9AB","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069110696","display_name":"Antonio Vergari","orcid":"https://orcid.org/0000-0003-0036-5678","observed_orcids":["https://orcid.org/0000-0003-0036-5678"]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Vergari, Antonio","raw_affiliation_strings":["School of Informatics , University of Edinburgh 10 Crichton Street , Edinburgh , EH8 9AB"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Informatics , University of Edinburgh 10 Crichton Street , Edinburgh , EH8 9AB","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5030503109","display_name":"Shay B. Cohen","orcid":"https://orcid.org/0000-0003-4753-8353","observed_orcids":["https://orcid.org/0000-0003-4753-8353"]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Cohen, Shay B.","raw_affiliation_strings":["School of Informatics , University of Edinburgh 10 Crichton Street , Edinburgh , EH8 9AB"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Informatics , University of Edinburgh 10 Crichton Street , Edinburgh , EH8 9AB","institution_ids":["https://openalex.org/I98677209"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I98677209"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9854000210762024,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9854000210762024,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.010300000198185444,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8679999709129333},{"id":"https://openalex.org/keywords/code-generation-evaluation","display_name":"code generation evaluation","score":0.7200000286102295},{"id":"https://openalex.org/keywords/prompt-augmentation","display_name":"prompt augmentation","score":0.574999988079071},{"id":"https://openalex.org/keywords/programming-languages","display_name":"programming languages","score":0.5189999938011169},{"id":"https://openalex.org/keywords/zero-shot-evaluation","display_name":"zero-shot evaluation","score":0.6330000162124634}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8219000101089478},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6771000027656555},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6158999800682068},{"id":"https://openalex.org/C26713055","wikidata":"https://www.wikidata.org/wiki/Q245962","display_name":"Implementation","level":2,"score":0.5033000111579895},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4408000111579895},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.43299999833106995},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.4287000000476837},{"id":"https://openalex.org/C117251300","wikidata":"https://www.wikidata.org/wiki/Q1849855","display_name":"Parametric statistics","level":2,"score":0.3977000117301941},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3578999936580658},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3571000099182129},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3522999882698059},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3409000039100647},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2703999876976013},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.26919999718666077},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.26330000162124634}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.02438","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.02438","pdf_url":"https://arxiv.org/pdf/2610.02438","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.02438","doi":"https://doi.org/10.48550/arxiv.2610.02438","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.02438","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.02438","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.02438","pdf_url":"https://arxiv.org/pdf/2610.02438","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320307875","display_name":"Styrelsen f\u00f6r Internationellt Utvecklingssamarbete","ror":"https://ror.org/01fn7me06"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220455017.pdf","grobid_xml":"https://content.openalex.org/works/W7220455017.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7214034969","https://openalex.org/W4378470905","https://openalex.org/W4411360072","https://openalex.org/W7137824411","https://openalex.org/W4417224607","https://openalex.org/W4411113095","https://openalex.org/W6904016667","https://openalex.org/W4412106628","https://openalex.org/W7202187487","https://openalex.org/W4400318591"],"abstract_inverted_index":{"Large":[0],"language":[1,155],"models":[2,109],"(LLMs)":[3],"have":[4],"demonstrated":[5],"strong":[6],"performance":[7],"in":[8,53,104,112,120],"code":[9,56,63,140,171,184],"generation,":[10],"where":[11],"success":[12],"depends":[13],"on":[14,148,163],"both":[15],"recalling":[16],"relevant":[17],"algorithmic":[18,144,183],"knowledge":[19,71],"and":[20,96,106,125,133,157,178,188],"reasoning":[21],"about":[22],"how":[23],"to":[24,100],"apply":[25],"it.":[26],"However,":[27],"existing":[28],"LLM":[29],"pipelines":[30],"are":[31,50,190],"opaque,":[32],"with":[33,138],"no":[34],"explicit":[35],"separation":[36],"between":[37],"these":[38],"two":[39],"components.":[40],"We":[41,78,116],"argue":[42],"that":[43,135],"for":[44,129],"well-known":[45],"algorithms":[46,89,132],"whose":[47],"canonical":[48],"implementations":[49],"widely":[51,130],"accessible":[52],"pretraining":[54],"corpora,":[55],"generation":[57],"is":[58],"better":[59],"measured":[60],"as":[61,173],"\\textit{parametric":[62],"retrieval}:":[64],"reproducing":[65],"a":[66,75,81,113,174],"named":[67],"algorithm":[68],"from":[69],"internalised":[70],"rather":[72],"than":[73],"synthesizing":[74],"novel":[76],"one.":[77],"introduce":[79],"AlgoREval,":[80],"benchmark":[82],"of":[83],"599":[84],"problems":[85],"spanning":[86],"classical":[87],"77":[88],"across":[90,123],"14":[91],"domains,":[92],"7":[93],"programming":[94],"languages,":[95],"4":[97],"graph-input":[98],"representations":[99],"evaluate":[101],"this":[102],"capability":[103,177],"isolation,":[105],"assess":[107],"15":[108],"(7B--34B":[110],"parameters)":[111],"zero-shot":[114],"setting.":[115],"find":[117],"substantial":[118],"variation":[119],"retrieval":[121,172],"accuracy":[122,147],"languages":[124],"input":[126],"representations,":[127],"even":[128],"documented":[131],"show":[134],"prompt":[136],"augmentation":[137],"retrieved":[139],"snippets":[141],"or":[142],"structured":[143],"hints":[145],"improve":[146],"complex":[149],"algorithms,":[150],"while":[151],"SFT":[152],"achieves":[153,159],"broader":[154],"gains":[156,162],"GRPO":[158],"larger":[160],"per-language":[161],"specific":[164],"languages.":[165],"Together,":[166],"our":[167],"results":[168],"establish":[169],"parametric":[170],"distinct,":[175],"measurable":[176],"caution":[179],"against":[180],"deploying":[181],"AI-generated":[182],"without":[185],"systematic":[186],"validation.\\footnote{Code":[187],"dataset":[189],"available":[191],"at":[192],"https://github.com/Nickil21/AlgoREval":[193]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-10-06T00:00:00"}