{"id":"https://openalex.org/W7093316558","doi":"https://doi.org/10.5281/zenodo.17423424","title":"Model See, Model Do? Exposure-Aware Evaluation of Bug-vs-Fix Preference in Code LLMs","display_name":"Model See, Model Do? Exposure-Aware Evaluation of Bug-vs-Fix Preference in Code LLMs","publication_year":2025,"publication_date":"2025-10-23","ids":{"openalex":"https://openalex.org/W7093316558","doi":"https://doi.org/10.5281/zenodo.17423424"},"language":null,"primary_location":{"id":"doi:10.5281/zenodo.17423424","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.17423424","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},"type":"software","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.17423424","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Al-Kaswan, Ali","orcid":"https://orcid.org/0000-0001-7338-2044"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Al-Kaswan, Ali","raw_affiliation_strings":["Delft University of Technology"],"raw_orcid":"https://orcid.org/0000-0001-7338-2044","affiliations":[{"raw_affiliation_string":"Delft University of Technology","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Spiess, Claudio","orcid":"https://orcid.org/0009-0000-5932-7022"},"institutions":[{"id":"https://openalex.org/I84218800","display_name":"University of California, Davis","ror":"https://ror.org/05rrcem69","country_code":"US","type":"education","lineage":["https://openalex.org/I84218800"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Spiess, Claudio","raw_affiliation_strings":["University of California, Davis"],"raw_orcid":"https://orcid.org/0009-0000-5932-7022","affiliations":[{"raw_affiliation_string":"University of California, Davis","institution_ids":["https://openalex.org/I84218800"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Devanbu, Prem","orcid":"https://orcid.org/0000-0002-4346-5276"},"institutions":[{"id":"https://openalex.org/I84218800","display_name":"University of California, Davis","ror":"https://ror.org/05rrcem69","country_code":"US","type":"education","lineage":["https://openalex.org/I84218800"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Devanbu, Prem","raw_affiliation_strings":["University of California, Davis"],"raw_orcid":"https://orcid.org/0000-0002-4346-5276","affiliations":[{"raw_affiliation_string":"University of California, Davis","institution_ids":["https://openalex.org/I84218800"]}]},{"author_position":"middle","author":{"id":null,"display_name":"van Deursen, Arie","orcid":"https://orcid.org/0000-0003-4850-3312"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"van Deursen, Arie","raw_affiliation_strings":["Delft University of Technology"],"raw_orcid":"https://orcid.org/0000-0003-4850-3312","affiliations":[{"raw_affiliation_string":"Delft University of Technology","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":null,"display_name":"Izadi, Maliheh","orcid":"https://orcid.org/0000-0001-5093-5523"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Izadi, Maliheh","raw_affiliation_strings":["Delft University of Technology"],"raw_orcid":"https://orcid.org/0000-0001-5093-5523","affiliations":[{"raw_affiliation_string":"Delft University of Technology","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.4593000113964081,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.4593000113964081,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.1421000063419342,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.07980000227689743,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/container","display_name":"Container (type theory)","score":0.7626000046730042},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.6064000129699707},{"id":"https://openalex.org/keywords/replication","display_name":"Replication (statistics)","score":0.5799000263214111},{"id":"https://openalex.org/keywords/clone","display_name":"clone (Java method)","score":0.5034999847412109},{"id":"https://openalex.org/keywords/directory","display_name":"Directory","score":0.4876999855041504},{"id":"https://openalex.org/keywords/replicate","display_name":"Replicate","score":0.4602000117301941},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.4397999942302704},{"id":"https://openalex.org/keywords/namespace","display_name":"Namespace","score":0.43950000405311584}],"concepts":[{"id":"https://openalex.org/C2781018962","wikidata":"https://www.wikidata.org/wiki/Q5164884","display_name":"Container (type theory)","level":2,"score":0.7626000046730042},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6674000024795532},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6064000129699707},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.5799000263214111},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.51910001039505},{"id":"https://openalex.org/C81089528","wikidata":"https://www.wikidata.org/wiki/Q5134986","display_name":"clone (Java method)","level":3,"score":0.5034999847412109},{"id":"https://openalex.org/C2777683733","wikidata":"https://www.wikidata.org/wiki/Q201456","display_name":"Directory","level":2,"score":0.4876999855041504},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.4602000117301941},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4397999942302704},{"id":"https://openalex.org/C2778383056","wikidata":"https://www.wikidata.org/wiki/Q873636","display_name":"Namespace","level":2,"score":0.43950000405311584},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.43050000071525574},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.3955000042915344},{"id":"https://openalex.org/C126831891","wikidata":"https://www.wikidata.org/wiki/Q221673","display_name":"Host (biology)","level":2,"score":0.37619999051094055},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.35530000925064087},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.34599998593330383},{"id":"https://openalex.org/C171730128","wikidata":"https://www.wikidata.org/wiki/Q5227290","display_name":"Data file","level":2,"score":0.3093000054359436},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.30160000920295715},{"id":"https://openalex.org/C47487241","wikidata":"https://www.wikidata.org/wiki/Q5227230","display_name":"Data access","level":2,"score":0.30090001225471497},{"id":"https://openalex.org/C122783720","wikidata":"https://www.wikidata.org/wiki/Q183065","display_name":"Interpreter","level":2,"score":0.2824999988079071},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.25940001010894775},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.25209999084472656},{"id":"https://openalex.org/C100463513","wikidata":"https://www.wikidata.org/wiki/Q5227322","display_name":"Data model (GIS)","level":2,"score":0.2502000033855438}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.5281/zenodo.17423424","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.17423424","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Software"}],"best_oa_location":{"id":"doi:10.5281/zenodo.17423424","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.17423424","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"This":[0,201],"repository":[1],"contains":[2],"the":[3,7,109,123,139,166,181,191,194,203,209,239,282,327,361,384,397],"replication":[4],"package":[5,51],"for":[6,39,55,95,211,288,300,308,316],"MSR'26":[8],"paper":[9],"titled":[10],"*\"Model":[11],"See,":[12],"Model":[13],"Do?":[14],"Exposure-Aware":[15],"Evaluation":[16],"of":[17,103],"Bug\u2013vs-Fix":[18],"Preference":[19],"in":[20,122,126,322,329,357,416],"Code":[21],"LLMs\"*.":[22],"It":[23],"provides":[24],"code,":[25],"data,":[26,263],"and":[27,46,60,81,117,128,216,248,266,291,351,367,389,421,427,438],"notebooks":[28,54,215,434],"to":[29,68,107,155,164,196,206,214,254,364,386,396,424,431,441],"replicate":[30],"our":[31],"experiments":[32,144,303],"on":[33,43,83],"evaluating":[34],"large":[35],"language":[36],"models":[37],"(LLMs)":[38],"code":[40],"generation,":[41],"focusing":[42],"bug-fix":[44],"preferences":[45],"data":[47,56,116,289,295,343,366,372],"exposure":[48],"awareness.":[49],"The":[50],"includes":[52],"Jupyter":[53],"preprocessing,":[57],"probability-based":[58],"experiments,":[59],"model":[61,96,270,310,318,346],"generation":[62],"analysis.":[63],"Intermediate":[64],"results":[65,119,334,414,426],"are":[66,120,335,394,415,439],"pre-provided":[67,321,413],"allow":[69],"skipping":[70],"computationally":[71],"intensive":[72],"steps.":[73],"Prerequisites":[74],"-":[75,86,112,259,272,279,285,297,305,313,354,359,379,382,392,405,418],"**Docker:**":[76],"Ensure":[77],"Docker":[78,154,182,283],"is":[79,93,105,162],"installed":[80],"running":[82],"your":[84,252],"system":[85],"**Hardware:**":[87],"A":[88,160],"machine":[89],"with":[90,294,371],"NVIDIA":[91],"GPUs":[92],"needed":[94],"inference":[97],"(via":[98],"vLLM).":[99],"At":[100],"least":[101],"100GBs":[102],"RAM":[104],"required":[106],"run":[108,146,220,368,422],"Data":[110],"Portraits.":[111],"**Data:**":[113],"All":[114,433],"necessary":[115],"intermediate":[118,264,333],"included":[121],"repo":[124],"(e.g.,":[125,245,268],"`/data`":[127],"`/plots`":[129],"folders)":[130],"If":[131,406],"you":[132,136,337],"don't":[133],"have":[134],"GPUs,":[135],"can":[137,338],"omit":[138],"`--gpus`":[140],"flag,":[141],"but":[142],"some":[143],"may":[145],"slowly":[147],"or":[148,345],"fail.":[149],"Setup":[150],"We":[151],"recommend":[152],"using":[153],"create":[156],"a":[157,198,243],"reproducible":[158],"environment.":[159,284],"`Dockerfile`":[161],"provided":[163],"build":[165,186],"image.":[167],"1.":[168,348],"Clone":[169],"this":[170],"repository:":[171],">":[172,176,184,218],"git":[173],"clone":[174],"https://github.com/your-username/your-repo-name.git":[175],"cd":[177],"your-repo-name":[178],"2.":[179],"Build":[180],"image:":[183],"docker":[185,219],"-t":[187],"deep_mem_cont:latest":[188,236],".":[189],"Running":[190],"Container":[192],"Run":[193,383,409],"container":[195,210],"start":[197],"JupyterLab":[199],"instance.":[200],"mounts":[202],"current":[204],"directory":[205],"`/home/jovyan/deep_mem`":[207],"inside":[208],"easy":[212],"access":[213,255],"data.":[217],"-p":[221],"10001:10001":[222],"--user":[223],"root":[224],"-e":[225,227],"CHOWN_HOME=yes":[226],"CHOWN_EXTRA_OPTS='-R'":[228],"--name":[229],"my-deep-mem-container":[230],"--rm":[231],"--gpus":[232],"'\"device=0,1,2\"'":[233],"-v":[234],"$(pwd):/home/jovyan/deep_mem":[235],"Once":[237],"running,":[238],"terminal":[240],"will":[241],"display":[242],"URL":[244],"`http://127.0.0.1:10001/?token=...`).":[246],"Copy":[247],"paste":[249],"it":[250,423],"into":[251],"browser":[253],"JupyterLab.":[256,358],"Repository":[257],"Structure":[258],"`data/`:":[260],"Contains":[261],"raw":[262],"results,":[265],"outputs":[267,319],"pre-generated":[269],"outputs).":[271],"`plots/`:":[273],"Stores":[274],"generated":[275],"plots":[276,429],"from":[277],"experiments.":[278],"`Dockerfile`:":[280],"Defines":[281],"`data_preprocessing.ipynb`:":[286],"Notebook":[287,299,307],"preprocessing":[290,344],"membership":[292,369],"testing":[293,370],"portraits.":[296,373],"`probs.ipynb`:":[298],"probability":[301],"metrics":[302,388],"(RQ2-3).":[304],"`gens.ipynb`:":[306],"analyzing":[309],"generations":[311],"(RQ4).":[312],"`vllm_gen.py`:":[314],"Script":[315],"generating":[317],"(results":[320],"`data/`).":[323,417],"Replicating":[324],"Experiments":[325,377,403],"Follow":[326,360],"instructions":[328,363],"each":[330],"notebook.":[331],"Since":[332],"pre-provided,":[336],"skip":[339],"resource-intensive":[340],"steps":[341],"like":[342],"inference.":[347],"**Data":[349],"Preprocessing":[350],"Membership":[352],"Testing:**":[353],"Open":[355,380,419],"`data_preprocessing.ipynb`":[356],"step-by-step":[362],"preprocess":[365],"3.":[374],"**Probability":[375],"Metrics":[376],"(RQ2-3):**":[378],"`probs.ipynb`.":[381],"notebook":[385],"compute":[387],"generate":[390,428],"plots.":[391],"Outputs":[393],"saved":[395],"`plots/`":[398],"folder.":[399],"4.":[400],"**Model":[401],"Generations":[402],"(RQ4):**":[404],"regenerating":[407],"outputs:":[408],"`vllm_gen.py`":[410],"(requires":[411],"GPUs;":[412],"`gens.ipynb`":[420],"analyze":[425],"(saved":[430],"`plots/`).":[432],"include":[435],"detailed":[436],"comments":[437],"designed":[440],"be":[442],"self-contained.":[443]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-24T00:00:00"}