{"id":"https://openalex.org/W7140294171","doi":"https://doi.org/10.48550/arxiv.2603.22474","title":"From Brittle to Robust: Improving LLM Annotations for SE Optimization","display_name":"From Brittle to Robust: Improving LLM Annotations for SE Optimization","publication_year":2026,"publication_date":"2026-03-23","ids":{"openalex":"https://openalex.org/W7140294171","doi":"https://doi.org/10.48550/arxiv.2603.22474"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.22474","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22474","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.22474","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5115763930","display_name":"Lohith Senthilkumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Senthilkumar, Lohith","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5130615836","display_name":"Tim Menzies","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Menzies, Tim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.2524999976158142,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.2524999976158142,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.08550000190734863,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.08190000057220459,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ensemble-learning","display_name":"Ensemble learning","score":0.5702000260353088},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5602999925613403},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5324000120162964},{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.48669999837875366},{"id":"https://openalex.org/keywords/ask-price","display_name":"Ask price","score":0.47839999198913574},{"id":"https://openalex.org/keywords/tree","display_name":"Tree (set theory)","score":0.4074999988079071},{"id":"https://openalex.org/keywords/hyperparameter","display_name":"Hyperparameter","score":0.39570000767707825},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.3882000148296356},{"id":"https://openalex.org/keywords/crossover","display_name":"Crossover","score":0.3873000144958496}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7558000087738037},{"id":"https://openalex.org/C45942800","wikidata":"https://www.wikidata.org/wiki/Q245652","display_name":"Ensemble learning","level":2,"score":0.5702000260353088},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5602999925613403},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5324000120162964},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5169000029563904},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5152999758720398},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.48669999837875366},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.47839999198913574},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.4074999988079071},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.39570000767707825},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.3882000148296356},{"id":"https://openalex.org/C122507166","wikidata":"https://www.wikidata.org/wiki/Q628906","display_name":"Crossover","level":2,"score":0.3873000144958496},{"id":"https://openalex.org/C169258074","wikidata":"https://www.wikidata.org/wiki/Q245748","display_name":"Random forest","level":2,"score":0.38609999418258667},{"id":"https://openalex.org/C119898033","wikidata":"https://www.wikidata.org/wiki/Q3433888","display_name":"Ensemble forecasting","level":2,"score":0.38199999928474426},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.34779998660087585},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.33570000529289246},{"id":"https://openalex.org/C135945739","wikidata":"https://www.wikidata.org/wiki/Q1211457","display_name":"Software release life cycle","level":5,"score":0.3346000015735626},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.3131999969482422},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.30709999799728394},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.3019999861717224},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.30160000920295715},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.29409998655319214},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.2791000008583069},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.2630000114440918},{"id":"https://openalex.org/C32587265","wikidata":"https://www.wikidata.org/wiki/Q1182260","display_name":"Data deduplication","level":2,"score":0.25839999318122864},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2529999911785126},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.2506999969482422}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.22474","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22474","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.22474","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.22474","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Software":[0],"analytics":[1],"often":[2],"builds":[3],"from":[4,109],"labeled":[5,175],"data.":[6],"Labeling":[7],"can":[8,193],"be":[9,210],"slow,":[10],"error":[11],"prone,":[12],"and":[13,139,165,212,226],"expensive.":[14],"When":[15,73],"human":[16,180],"expertise":[17],"is":[18,52],"scarce,":[19],"SE":[20,125],"researchers":[21],"sometimes":[22],"ask":[23],"large":[24],"language":[25],"models":[26],"(LLMs)":[27],"for":[28,55],"the":[29],"missing":[30],"labels.":[31],"While":[32],"this":[33,62],"has":[34,46,120],"been":[35,121],"successful":[36,205],"in":[37,162],"some":[38],"domains,":[39],"recent":[40],"results":[41,108,208],"show":[42],"that":[43,146,187,203],"LLM-based":[44],"labeling":[45,51],"blind":[47],"spots.":[48],"Specifically,":[49],"their":[50],"not":[53],"effective":[54],"higher":[56],"dimensional":[57,197],"multi-objective":[58,126,198],"problems.":[59],"To":[60,219],"address":[61],"task,":[63],"we":[64,185,201],"propose":[65],"a":[66],"novel":[67],"LLM":[68],"prompting":[69,207],"strategy":[70],"called":[71],"SynthCore.":[72],"one":[74],"opinion":[75],"fails,":[76],"SynthCore's":[77,142,216],"combines":[78],"multiple":[79,110],"separated":[80],"opinions":[81],"generated":[82],"by":[83,176],"LLMs":[84],"(with":[85,114],"no":[86,115],"knowledge":[87],"of":[88,95,157,189],"each":[89],"others'":[90],"answers)":[91],"into":[92],"an":[93],"ensemble":[94,143,217],"few-shot":[96,206],"learners.":[97],"Simpler":[98],"than":[99,149],"other":[100,204],"strategies":[101],"(e.g.":[102],"chain-of-thought,":[103],"multi-agent-debate,":[104],"etc)":[105],"SynthCore":[106,119],"aggregates":[107],"single":[111],"prompt":[112],"sessions":[113],"crossover":[116],"between":[117],"them).":[118],"tested":[122],"on":[123],"49":[124],"optimization":[127],"tasks,":[128],"handling":[129],"tasks":[130],"as":[131,133],"diverse":[132],"software":[134],"project":[135],"management,":[136],"Makefile":[137],"configuration,":[138],"hyperparameter":[140],"optimization.":[141],"found":[144],"optimizations":[145,170],"are":[147,228],"better":[148],"state-of-the-art":[150],"alternative":[151],"approaches":[152],"(Gaussian":[153],"Process":[154],"Models,":[155],"Tree":[156],"Parzen":[158],"Estimators,":[159],"active":[160],"learners":[161,192],"both":[163],"exploration":[164],"exploitation":[166],"mode).":[167],"Importantly,":[168],"these":[169,183],"were":[171],"made":[172],"using":[173,215],"data":[174,225],"LLMs,":[177],"without":[178],"any":[179],"opinions.":[181],"From":[182],"experiments,":[184],"conclude":[186],"ensembles":[188],"few":[190],"shot":[191],"successfully":[194],"annotate":[195],"high":[196],"tasks.":[199],"Further,":[200],"speculate":[202],"could":[209],"quickly":[211],"easily":[213],"enhanced":[214],"approach.":[218],"support":[220],"open":[221],"science,":[222],"all":[223],"our":[224],"scripts":[227],"available":[229],"at":[230],"https://github.com/lohithsowmiyan/lazy-llm/tree/clusters.":[231]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-03-26T00:00:00"}