{"id":"https://openalex.org/W7133358418","doi":"https://doi.org/10.48550/arxiv.2603.01051","title":"CelloAI Benchmarks: Toward Repeatable Evaluation of AI Assistants","display_name":"CelloAI Benchmarks: Toward Repeatable Evaluation of AI Assistants","publication_year":2026,"publication_date":"2026-03-01","ids":{"openalex":"https://openalex.org/W7133358418","doi":"https://doi.org/10.48550/arxiv.2603.01051"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.01051","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.01051","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.01051","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5127908593","display_name":"Mohammad Atif","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Atif, Mohammad","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127975867","display_name":"Kriti Chopra","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chopra, Kriti","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128005797","display_name":"Fang-Ying Tsai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tsai, Fang-Ying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128034977","display_name":"Ozgur O. Kilic","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kilic, Ozgur O.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127962552","display_name":"Tianle Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Tianle","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127894501","display_name":"Zhihua Dong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dong, Zhihua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127922315","display_name":"Douglas Benjamin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Benjamin, Douglas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127919385","display_name":"Charles Leggett","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Leggett, Charles","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128007516","display_name":"Meifeng Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Meifeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5127905556","display_name":"Paolo Calafiura","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Calafiura, Paolo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5127882926","display_name":"Salman Habib","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Habib, Salman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.2736000120639801,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.2736000120639801,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.267300009727478,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.0575999990105629,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.7419999837875366},{"id":"https://openalex.org/keywords/documentation","display_name":"Documentation","score":0.6777999997138977},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.5702000260353088},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.5353999733924866},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4966999888420105},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.4724999964237213},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.45910000801086426},{"id":"https://openalex.org/keywords/software-inspection","display_name":"Software inspection","score":0.45820000767707825}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8119999766349792},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.7419999837875366},{"id":"https://openalex.org/C56666940","wikidata":"https://www.wikidata.org/wiki/Q788790","display_name":"Documentation","level":2,"score":0.6777999997138977},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.6147000193595886},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.5702000260353088},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.5353999733924866},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4966999888420105},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.4724999964237213},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.45910000801086426},{"id":"https://openalex.org/C10272871","wikidata":"https://www.wikidata.org/wiki/Q929972","display_name":"Software inspection","level":5,"score":0.45820000767707825},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.4415000081062317},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.42570000886917114},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4239000082015991},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.4083999991416931},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.3986000120639801},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.39640000462532043},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.37869998812675476},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.32109999656677246},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.3163999915122986},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2856000065803528},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.27390000224113464},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.27239999175071716},{"id":"https://openalex.org/C2777561058","wikidata":"https://www.wikidata.org/wiki/Q2652119","display_name":"Program comprehension","level":4,"score":0.2711000144481659},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.25859999656677246}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.01051","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.01051","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.01051","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.01051","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLM)":[3],"are":[4],"increasingly":[5],"used":[6],"for":[7,13,116,160],"software":[8,162],"development,":[9],"yet":[10],"existing":[11],"benchmarks":[12,65,81,94,106,111],"LLM-based":[14],"coding":[15,121],"assistance":[16,122],"do":[17],"not":[18],"reflect":[19],"the":[20,60,83,142],"constraints":[21,38],"of":[22,56,62,85,147],"High":[23,28],"Energy":[24],"Physics":[25],"(HEP)":[26],"and":[27,39,50,102,129,138,149],"Performance":[29],"Computing":[30],"(HPC)":[31],"software.":[32],"Code":[33],"correctness":[34],"must":[35,41],"respect":[36],"science":[37],"changes":[40],"integrate":[42],"into":[43],"large,":[44],"performance-critical":[45],"codebases":[46],"with":[47],"complex":[48],"dependencies":[49],"build":[51],"systems.":[52],"The":[53],"primary":[54],"contribution":[55],"this":[57],"paper":[58],"is":[59],"development":[61],"practical,":[63],"repeatable":[64],"that":[66,157],"quantify":[67],"LLM":[68,87],"performance":[69],"on":[70,98,155],"HEP/HPC-relevant":[71],"tasks.":[72],"We":[73],"introduce":[74],"three":[75],"evaluation":[76],"tracks":[77],"--":[78],"code":[79,92,126],"documentation":[80,124],"measure":[82],"ability":[84],"an":[86],"to":[88],"generate":[89],"Doxygen-style":[90],"comments,":[91],"generation":[93,127],"evaluate":[95,107],"end-to-end":[96],"usability":[97],"representative":[99],"GPU":[100],"kernels,":[101],"graphical":[103],"data":[104],"analysis":[105],"vision-enabled":[108],"LLMs.":[109],"These":[110],"provide":[112],"a":[113],"unified":[114],"framework":[115],"measuring":[117],"progress":[118],"in":[119],"scientific":[120],"across":[123],"quality,":[125],"robustness,":[128],"multimodal":[130],"validation":[131],"analysis.":[132],"By":[133],"emphasizing":[134],"repeatability,":[135],"automated":[136],"scoring,":[137],"domain-relevant":[139],"failure":[140],"modes,":[141],"suite":[143],"enables":[144],"fair":[145],"comparisons":[146],"models":[148],"settings":[150],"while":[151],"supporting":[152],"future":[153],"work":[154],"methods":[156],"improve":[158],"reliability":[159],"HEP/HPC":[161],"development.":[163]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-03-04T00:00:00"}