{"id":"https://openalex.org/W7124965845","doi":"https://doi.org/10.1109/asew67777.2025.00015","title":"Vintage Code, Modern Judges: Meta-Validation in Low Data Regimes","display_name":"Vintage Code, Modern Judges: Meta-Validation in Low Data Regimes","publication_year":2025,"publication_date":"2025-11-16","ids":{"openalex":"https://openalex.org/W7124965845","doi":"https://doi.org/10.1109/asew67777.2025.00015"},"language":null,"primary_location":{"id":"doi:10.1109/asew67777.2025.00015","is_oa":false,"landing_page_url":"https://doi.org/10.1109/asew67777.2025.00015","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 40th IEEE/ACM International Conference on Automated Software Engineering Workshops (ASEW)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5123424714","display_name":"Ora Fandina","orcid":null},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":true,"raw_author_name":"Ora Fandina","raw_affiliation_strings":["IBM Research,Israel"],"affiliations":[{"raw_affiliation_string":"IBM Research,Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027466010","display_name":"Gal Amram","orcid":"https://orcid.org/0000-0003-2138-7542"},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Gal Amram","raw_affiliation_strings":["IBM Research,Israel"],"affiliations":[{"raw_affiliation_string":"IBM Research,Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064288943","display_name":"Eitan Farchi","orcid":"https://orcid.org/0000-0002-3021-1488"},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Eitan Farchi","raw_affiliation_strings":["IBM Research,Israel"],"affiliations":[{"raw_affiliation_string":"IBM Research,Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123429415","display_name":"Shmulik Froimovich","orcid":null},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Shmulik Froimovich","raw_affiliation_strings":["IBM Research,Israel"],"affiliations":[{"raw_affiliation_string":"IBM Research,Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123390001","display_name":"Raviv Gal Wesam Ibraheem","orcid":null},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Raviv Gal Wesam Ibraheem","raw_affiliation_strings":["IBM Research,Israel"],"affiliations":[{"raw_affiliation_string":"IBM Research,Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114320330","display_name":"Rami Katan","orcid":null},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Rami Katan","raw_affiliation_strings":["IBM Research,Israel"],"affiliations":[{"raw_affiliation_string":"IBM Research,Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Alice Podolsky","orcid":null},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Alice Podolsky","raw_affiliation_strings":["IBM Research,Israel"],"affiliations":[{"raw_affiliation_string":"IBM Research,Israel","institution_ids":["https://openalex.org/I4210167297"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043559592","display_name":"Orna Raz","orcid":null},"institutions":[{"id":"https://openalex.org/I4210167297","display_name":"IBM Research - Haifa","ror":"https://ror.org/05rw9t746","country_code":"IL","type":"facility","lineage":["https://openalex.org/I1341412227","https://openalex.org/I4210114115","https://openalex.org/I4210167297"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Orna Raz","raw_affiliation_strings":["IBM Research,Israel"],"affiliations":[{"raw_affiliation_string":"IBM Research,Israel","institution_ids":["https://openalex.org/I4210167297"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5123424714"],"corresponding_institution_ids":["https://openalex.org/I4210167297"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.87153206,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"30","last_page":"33"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.1143999993801117,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.1143999993801117,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11450","display_name":"Model-Driven Software Engineering Techniques","score":0.07999999821186066,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.07930000126361847,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.5601000189781189},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.5248000025749207},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.4697999954223633},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.45249998569488525},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.39419999718666077},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.3596000075340271},{"id":"https://openalex.org/keywords/statistical-model","display_name":"Statistical model","score":0.35190001130104065},{"id":"https://openalex.org/keywords/cobol","display_name":"COBOL","score":0.3450999855995178},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.34139999747276306}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7378000020980835},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.5601000189781189},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5248000025749207},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.4697999954223633},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4641999900341034},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4555000066757202},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.45249998569488525},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.39419999718666077},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3856000006198883},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38260000944137573},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.36039999127388},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.3596000075340271},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.35190001130104065},{"id":"https://openalex.org/C128595289","wikidata":"https://www.wikidata.org/wiki/Q131140","display_name":"COBOL","level":2,"score":0.3450999855995178},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.34139999747276306},{"id":"https://openalex.org/C72161134","wikidata":"https://www.wikidata.org/wiki/Q5421219","display_name":"Expert elicitation","level":2,"score":0.31380000710487366},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.3082999885082245},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.30660000443458557},{"id":"https://openalex.org/C188087704","wikidata":"https://www.wikidata.org/wiki/Q369577","display_name":"Standardization","level":2,"score":0.28690001368522644},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.274399995803833},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.2678000032901764},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.267300009727478},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2648000121116638},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.263700008392334},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.2549000084400177},{"id":"https://openalex.org/C53844881","wikidata":"https://www.wikidata.org/wiki/Q856122","display_name":"Modernization theory","level":2,"score":0.2547000050544739},{"id":"https://openalex.org/C153876917","wikidata":"https://www.wikidata.org/wiki/Q899704","display_name":"Traceability","level":2,"score":0.25440001487731934},{"id":"https://openalex.org/C108713360","wikidata":"https://www.wikidata.org/wiki/Q1824206","display_name":"Supply chain","level":2,"score":0.2533999979496002},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/asew67777.2025.00015","is_oa":false,"landing_page_url":"https://doi.org/10.1109/asew67777.2025.00015","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 40th IEEE/ACM International Conference on Automated Software Engineering Workshops (ASEW)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":3,"referenced_works":["https://openalex.org/W4401905935","https://openalex.org/W4411233288","https://openalex.org/W7120088840"],"related_works":[],"abstract_inverted_index":{"Application":[0],"modernization":[1],"in":[2,18,22,50],"legacy":[3],"languages":[4],"such":[5],"as":[6,31],"COBOL,":[7],"PL/I,":[8],"and":[9,21,75,98,159],"REXX":[10],"faces":[11],"an":[12],"acute":[13],"shortage":[14],"of":[15,109],"resources,":[16],"both":[17],"expert":[19,40],"availability":[20,108],"high-quality":[23],"human":[24,92,127],"evaluation":[25,60,111],"data.":[26,142],"While":[27],"Large":[28],"Language":[29],"Models":[30],"a":[32,36,58,95,123,132,145,150],"Judge":[33],"(LaaJ)":[34],"offer":[35],"scalable":[37],"alternative":[38],"to":[39,67,84,119,174,182],"review,":[41],"their":[42],"reliability":[43],"must":[44],"be":[45],"validated":[46],"before":[47],"being":[48],"trusted":[49],"high-stakes":[51],"workflows.":[52],"Without":[53],"principled":[54],"validation,":[55],"organizations":[56],"risk":[57],"circular":[59],"loop,":[61],"where":[62],"unverified":[63],"LaaJs":[64,86,184],"are":[65,171],"used":[66,97],"assess":[68,120],"model":[69,198],"outputs,":[70],"potentially":[71],"reinforcing":[72],"unreliable":[73],"judgments":[74],"compromising":[76],"downstream":[77],"deployment":[78],"decisions.":[79,200],"Although":[80],"various":[81],"automated":[82],"approaches":[83],"validating":[85],"have":[87],"been":[88],"proposed,":[89],"alignment":[90,138,152],"with":[91,126,139,149],"judgment":[93],"remains":[94],"widely":[96],"conceptually":[99],"grounded":[100],"validation":[101],"strategy.":[102],"In":[103],"many":[104],"real-world":[105],"domains,":[106],"the":[107],"human-labeled":[110,141],"data":[112],"is":[113],"severely":[114],"limited,":[115],"making":[116],"it":[117],"difficult":[118],"how":[121],"well":[122],"LaaJ":[124,137],"aligns":[125],"judgment.":[128],"We":[129],"introduce":[130],"SparseAlign,":[131],"formal":[133],"framework":[134],"for":[135,185],"assessing":[136],"sparse":[140],"SparseAlign":[143,178],"combines":[144],"novel":[146],"pairwise-confidence":[147],"concept":[148],"scoresensitive":[151],"metric":[153],"that":[154],"jointly":[155],"capture":[156],"ranking":[157],"consistency":[158],"score":[160],"proximity,":[161],"enabling":[162],"reliable":[163],"evaluator":[164],"selection":[165],"even":[166],"when":[167],"traditional":[168],"statistical":[169],"methods":[170],"ineffective":[172],"due":[173],"limited":[175],"annotated":[176],"examples.":[177],"was":[179],"applied":[180],"internally":[181],"select":[183],"COBOL":[186],"code":[187],"explanation.":[188],"The":[189],"top-aligned":[190],"evaluators":[191],"were":[192],"integrated":[193],"into":[194],"assessment":[195],"workflows,":[196],"guiding":[197],"release":[199]},"counts_by_year":[],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2026-01-21T00:00:00"}