{"id":"https://openalex.org/W7162311116","doi":"https://doi.org/10.48550/arxiv.2605.23008","title":"On the Reliability of Code Comprehension Proxies","display_name":"On the Reliability of Code Comprehension Proxies","publication_year":2026,"publication_date":"2026-05-21","ids":{"openalex":"https://openalex.org/W7162311116","doi":"https://doi.org/10.48550/arxiv.2605.23008"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.23008","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.23008","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.23008","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5090755115","display_name":"Erfan Arvan","orcid":"https://orcid.org/0000-0002-6170-1957"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Arvan, Erfan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046232249","display_name":"Nadeeshan De Silva","orcid":"https://orcid.org/0000-0001-5325-9030"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"De Silva, Nadeeshan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003334072","display_name":"Oscar Chaparro","orcid":"https://orcid.org/0000-0003-2838-685X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chaparro, Oscar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5043023022","display_name":"Martin Kellogg","orcid":"https://orcid.org/0000-0002-3185-2340"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kellogg, Martin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.8708999752998352,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.8708999752998352,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.07760000228881836,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.008500000461935997,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/comprehension","display_name":"Comprehension","score":0.6186000108718872},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.612500011920929},{"id":"https://openalex.org/keywords/program-comprehension","display_name":"Program comprehension","score":0.5670999884605408},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.5311999917030334},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5266000032424927},{"id":"https://openalex.org/keywords/syntax","display_name":"Syntax","score":0.5214999914169312},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.4876999855041504},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.4657000005245209}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6815999746322632},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.6186000108718872},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.612500011920929},{"id":"https://openalex.org/C2777561058","wikidata":"https://www.wikidata.org/wiki/Q2652119","display_name":"Program comprehension","level":4,"score":0.5670999884605408},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.5311999917030334},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5266000032424927},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.5214999914169312},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.4876999855041504},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4657000005245209},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4528999924659729},{"id":"https://openalex.org/C2779495148","wikidata":"https://www.wikidata.org/wiki/Q487378","display_name":"Delphi","level":2,"score":0.3991999924182892},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.3846000134944916},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36890000104904175},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.359499990940094},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.35010001063346863},{"id":"https://openalex.org/C82214349","wikidata":"https://www.wikidata.org/wiki/Q657339","display_name":"Software metric","level":5,"score":0.3398999869823456},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.32589998841285706},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.3206000030040741},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3124000132083893},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.3116999864578247},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.30820000171661377},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3077000081539154},{"id":"https://openalex.org/C60641444","wikidata":"https://www.wikidata.org/wiki/Q841602","display_name":"Delphi method","level":2,"score":0.29670000076293945},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2565000057220459},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.2500999867916107}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.23008","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.23008","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.23008","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.23008","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6645303964614868}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Prior":[0],"work":[1],"on":[2,152,166],"code":[3,27,87,158],"comprehension":[4,7,146],"uses":[5],"different":[6],"proxies-for":[8],"example,":[9],"Likert-scale":[10],"ratings":[11],"or":[12],"answers":[13],"to":[14,24,30,79,118,143],"input-output":[15,175],"questions":[16,176,196],"about":[17,197],"program":[18,198],"snippets,":[19],"usually":[20],"collected":[21],"from":[22,149,174,195],"students,":[23],"approximate":[25],"whether":[26],"is":[28,40,98,122],"comprehensible":[29],"software":[31,77,127],"engineers,":[32],"but":[33,117],"the":[34,46,56,91,150,153,167,214,219],"relative":[35,47],"reliability":[36,48,215],"of":[37,49,52,62,74,85,106,156,207,216,218],"these":[38],"proxies":[39,53,147,171,193],"not":[41],"known.":[42],"This":[43],"paper":[44],"investigates":[45],"a":[50,60,72,81,132,163],"collection":[51],"common":[54],"in":[55,108,126],"extant":[57],"literature":[58,151],"with":[59,71,134],"pair":[61],"human":[63],"studies.":[64],"First,":[65],"we":[66,130,161],"conducted":[67,131,162],"an":[68],"expert-consensus":[69,93],"study":[70,133],"panel":[73],"five":[75],"professional":[76],"engineers":[78],"establish":[80],"ground-truth":[82],"comprehensibility":[83,221],"ranking":[84],"eight":[86,157],"snippets":[88],"by":[89],"adapting":[90],"Delphi":[92,96],"protocol.":[94],"The":[95],"protocol":[97],"widely":[99],"used":[100],"for":[101],"expert":[102],"consensus":[103],"under":[104],"conditions":[105],"uncertainty":[107],"other":[109],"domains,":[110],"such":[111],"as":[112],"medicine":[113],"and":[114,177],"national-security":[115],"forecasting,":[116],"our":[119],"knowledge,":[120],"this":[121],"its":[123],"first":[124],"application":[125],"engineering.":[128],"Second,":[129],"44":[135],"student":[136],"participants":[137],"who":[138],"completed":[139],"tasks,":[140],"allowing":[141],"us":[142],"measure":[144,180],"14":[145],"derived":[148,173,194],"same":[154],"set":[155],"snippets.":[159],"Finally,":[160],"correlation":[164],"analysis":[165],"results,":[168],"concluding":[169],"that":[170,179,192],"1)":[172],"2)":[178],"response":[181],"time":[182],"rather":[183],"than":[184,201],"accuracy":[185],"are":[186,203],"especially":[187,204],"reliable.":[188],"We":[189],"also":[190],"found":[191],"syntax":[199],"(rather":[200],"semantics)":[202],"unreliable,":[205],"regardless":[206],"measurement":[208],"strategy,":[209],"which":[210],"draws":[211],"into":[212],"question":[213],"parts":[217],"existing":[220],"literature.":[222]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-05-26T00:00:00"}