{"id":"https://openalex.org/W7128682121","doi":"https://doi.org/10.5281/zenodo.18621369","title":"Replication Package for the Paper: Evaluating Large Language Models for Detecting Architectural Decision Violations","display_name":"Replication Package for the Paper: Evaluating Large Language Models for Detecting Architectural Decision Violations","publication_year":2026,"publication_date":"2026-02-12","ids":{"openalex":"https://openalex.org/W7128682121","doi":"https://doi.org/10.5281/zenodo.18621369"},"language":null,"primary_location":{"id":"pmh:doi:10.5281/zenodo.17799431","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},"type":"dataset","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125698364","display_name":"Ruoyu Su","orcid":null},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Su, Ruoyu","raw_affiliation_strings":["University of Oulu"],"raw_orcid":"https://orcid.org/0009-0008-6206-8787","affiliations":[{"raw_affiliation_string":"University of Oulu","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078111792","display_name":"Alexander Bakhtin","orcid":"https://orcid.org/0000-0003-3513-7253"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Bakhtin, Alexander","raw_affiliation_strings":["University of Oulu"],"raw_orcid":"https://orcid.org/0000-0003-3513-7253","affiliations":[{"raw_affiliation_string":"University of Oulu","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113423845","display_name":"Noman Ahmad","orcid":null},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Ahmad, Noman","raw_affiliation_strings":["University of Oulu"],"raw_orcid":"https://orcid.org/0009-0005-4228-2493","affiliations":[{"raw_affiliation_string":"University of Oulu","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125694421","display_name":"Matteo Esposito","orcid":null},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Esposito, Matteo","raw_affiliation_strings":["University of Oulu"],"raw_orcid":"https://orcid.org/0000-0002-8451-3668","affiliations":[{"raw_affiliation_string":"University of Oulu","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015576503","display_name":"Valentina Lenarduzzi","orcid":"https://orcid.org/0000-0003-0511-5133"},"institutions":[{"id":"https://openalex.org/I177969490","display_name":"University of Southern Denmark","ror":"https://ror.org/03yrrjy16","country_code":"DK","type":"education","lineage":["https://openalex.org/I177969490"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Lenarduzzi, Valentina","raw_affiliation_strings":["University of Southern Denmark"],"raw_orcid":"https://orcid.org/0000-0003-0511-5133","affiliations":[{"raw_affiliation_string":"University of Southern Denmark","institution_ids":["https://openalex.org/I177969490"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5123004219","display_name":"Davide Taibi","orcid":null},"institutions":[{"id":"https://openalex.org/I177969490","display_name":"University of Southern Denmark","ror":"https://ror.org/03yrrjy16","country_code":"DK","type":"education","lineage":["https://openalex.org/I177969490"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Taibi, Davide","raw_affiliation_strings":["University of Southern Denmark"],"raw_orcid":"https://orcid.org/0000-0002-3210-3990","affiliations":[{"raw_affiliation_string":"University of Southern Denmark","institution_ids":["https://openalex.org/I177969490"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.6855000257492065},{"id":"https://openalex.org/keywords/data-file","display_name":"Data file","score":0.531000018119812},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.5055000185966492},{"id":"https://openalex.org/keywords/jaccard-index","display_name":"Jaccard index","score":0.39719998836517334},{"id":"https://openalex.org/keywords/replication","display_name":"Replication (statistics)","score":0.3961000144481659},{"id":"https://openalex.org/keywords/codebase","display_name":"Codebase","score":0.3952000141143799},{"id":"https://openalex.org/keywords/directory","display_name":"Directory","score":0.3456999957561493},{"id":"https://openalex.org/keywords/file-format","display_name":"File format","score":0.3456000089645386},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.3269999921321869}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8216999769210815},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.6855000257492065},{"id":"https://openalex.org/C171730128","wikidata":"https://www.wikidata.org/wiki/Q5227290","display_name":"Data file","level":2,"score":0.531000018119812},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.5055000185966492},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.49140000343322754},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.47119998931884766},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4047999978065491},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.40380001068115234},{"id":"https://openalex.org/C203519979","wikidata":"https://www.wikidata.org/wiki/Q865360","display_name":"Jaccard index","level":3,"score":0.39719998836517334},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.3961000144481659},{"id":"https://openalex.org/C51929080","wikidata":"https://www.wikidata.org/wiki/Q2425187","display_name":"Codebase","level":3,"score":0.3952000141143799},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.35030001401901245},{"id":"https://openalex.org/C2777683733","wikidata":"https://www.wikidata.org/wiki/Q201456","display_name":"Directory","level":2,"score":0.3456999957561493},{"id":"https://openalex.org/C97250363","wikidata":"https://www.wikidata.org/wiki/Q235557","display_name":"File format","level":2,"score":0.3456000089645386},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.3269999921321869},{"id":"https://openalex.org/C2780560020","wikidata":"https://www.wikidata.org/wiki/Q79719","display_name":"License","level":2,"score":0.30959999561309814},{"id":"https://openalex.org/C92446256","wikidata":"https://www.wikidata.org/wiki/Q3306762","display_name":"Data validation","level":2,"score":0.3075000047683716},{"id":"https://openalex.org/C56666940","wikidata":"https://www.wikidata.org/wiki/Q788790","display_name":"Documentation","level":2,"score":0.3061000108718872},{"id":"https://openalex.org/C1668388","wikidata":"https://www.wikidata.org/wiki/Q1149776","display_name":"Data management","level":2,"score":0.30070000886917114},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.2863999903202057},{"id":"https://openalex.org/C5274069","wikidata":"https://www.wikidata.org/wiki/Q2285707","display_name":"Categorical variable","level":2,"score":0.2842000126838684},{"id":"https://openalex.org/C43364308","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Byte","level":2,"score":0.2784000039100647},{"id":"https://openalex.org/C174183944","wikidata":"https://www.wikidata.org/wiki/Q334661","display_name":"MIT License","level":3,"score":0.266400009393692},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.2635999917984009},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2606000006198883},{"id":"https://openalex.org/C52146309","wikidata":"https://www.wikidata.org/wiki/Q7431116","display_name":"Schema (genetic algorithms)","level":2,"score":0.2565000057220459},{"id":"https://openalex.org/C154504017","wikidata":"https://www.wikidata.org/wiki/Q853614","display_name":"Identifier","level":2,"score":0.2526000142097473}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.5281/zenodo.17799431","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},{"id":"doi:10.5281/zenodo.18621369","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.18621369","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Dataset"}],"best_oa_location":{"id":"pmh:doi:10.5281/zenodo.17799431","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"This":[0,102,536,675,1200,1255],"replication":[1],"package":[2],"provides":[3],"how":[4,117],"we":[5,1064,1073,1105,1111,1133,1151,1157,1183],"evaluate":[6],"LLMs":[7,342,359,519,553,1146,1161],"for":[8,34,138,179,233,339,447,505,513,665,757,934,1079,1204,1378],"detecting":[9],"ADR":[10,18,32,196,210,227,300,307,320,333,352,363,507,546,549,819,833,858,898,909,931,941,954,962],"Violations.":[11],"The":[12,65,110,169,184,592,706,744,779,804,835,924,958,1228,1327,1345],"packge":[13],"leverages":[14],"the":[15,27,35,42,48,52,58,62,72,77,97,106,114,120,124,128,139,162,180,189,199,220,230,235,240,261,264,286,291,297,324,331,336,351,362,372,375,382,398,413,448,458,462,484,488,498,506,514,529,534,539,544,560,563,570,577,580,590,598,602,612,617,640,643,652,656,662,666,670,684,687,691,696,700,711,714,718,727,749,764,769,772,782,798,809,818,822,827,832,838,845,867,872,877,901,908,920,937,944,947,952,956,972,981,1005,1008,1040,1046,1055,1058,1066,1070,1077,1089,1092,1095,1107,1113,1122,1126,1135,1139,1153,1164,1172,1175,1178,1185,1211,1291,1299,1333],"dataset":[16,171,190,201,242,265,298,829],"of":[17,105,146,154,161,188,222,249,251,288,290,299,533,699,774,776,784,800,883,1116,1235],"files":[19,33,54,172,197,211,228,272,301,308,321,334,689,721,728,745,789,820,843,899,910,932,942,963],"from":[20,174,198,213,229,263,273,309,366,407,426,434,441,555,589,787,844,936,1155,1188],"Buchgeher":[21],"et":[22],"al.":[23],"(ORIGINAL_README.md)":[24],"to":[25,51,118,122,258,268,282,295,302,329,335,346,349,356,360,370,374,380,396,457,460,483,486,542,567,583,585,597,635,650,658,694,748,767,796,808,825,900,919,943,950,955,968,980,1031,1053,1091,1120,1162,1174,1210],"prepare":[26],"project's":[28],"codebase":[29,688,750,788,945,983,1242],"and":[30,41,116,126,132,219,247,285,314,390,419,471,548,572,611,642,683,732,736,771,793,830,848,856,863,881,887,892,916,978,1049,1086,1124,1169,1193,1337,1382],"selected":[31,215,231,332,933,953],"analysis":[36,129,340,935],"with":[37,341,475,1069,1094,1177],"Large":[38],"Language":[39],"Models,":[40],"LLM":[43,164,530,1252],"Detection":[44],"Process.":[45],"Note:":[46],"All":[47,80,92],"hyperlinks":[49],"referring":[50],"shared":[53],"only":[55,66,212,896,907],"work":[56,70,373,1090,1173],"in":[57,203,239,565,713,733,791,802,813,890,965,971,976,984,1024,1039,1244,1372],"local":[59,1265],"version":[60,67],"(downloading":[61],"relpication":[63],"package).":[64],"will":[68,672,927],"not":[69,75,620,990,1207],"as":[71,316,318,528,661,1057,1076],"browser":[73],"does":[74],"find":[76,969],"files.":[78,705,834],"License":[79],"generated":[81,406,425,433,440,517],"data":[82,125,455,467,481,493,805,894,996,1068,1224,1230],"is":[83,607,614,619,629,677,806,989,993,1130,1143,1202,1294],"provided":[84,95],"under":[85,96],"DATA_LICENSE":[86],"Creative":[87],"Commons":[88],"4.0":[89],"Attribution":[90],"License.":[91,100],"scripts":[93,121,593,1001],"are":[94,646,729,746,1002],"Script_LICENSE":[98],"MIT":[99],"Contents":[101],"repository":[103,191,613,682,853,975],"consists":[104],"following":[107,1009,1096],"files:":[108],"README.md:":[109],"README":[111,186],"file":[112,187,279,811,869,903,922,925,988,1318],"describes":[113,538],"project":[115,253,644,657,667,761,1256],"run":[119,1298],"get":[123,1065,1106,1134,1152,1184],"do":[127,1206],"INSTALL.md:":[130],"Installation":[131],"configurations":[133],"instructions":[134,564],"requirements.txt:":[135],"Library":[136],"requirements":[137],"Python":[140],"environment":[141,182,561,571,574,604,755,815],"Figures":[142],"jaccard_C_heatmap.pdf:":[143],"Heatmap":[144,152],"diagram":[145,153],"pairwise":[147,155,391],"jaccard":[148,156],"coefficients":[149,157],"(C)":[150],"jaccard_overall_heatmap.pdf:":[151],"(overall)":[158],"llm_process.pdf:":[159],"Diagram":[160],"general":[163],"Processing":[165],"Implementation":[166],"Data_collection":[167],"ADR-dataset:":[168],"original":[170,185,200,241,828,846,938],"(renamed":[173],"repositories)":[175],".env:":[176],"Environment":[177],"variables":[178],"bash":[181],"ORIGINAL_README.md:":[183],"adr_all.csv:":[192],"Data":[193,208,225,244],"on":[194,209,217,226,245,312,323,851,876,1218,1340],"all":[195,304,497,587,595,679,710,717,842,930,961,1000],"reported":[202],"a":[204,276,740,987,1215,1219,1264],"single":[205,277,741,1220],"CSV":[206,810,868,902,921,1317],"adr_selected_projects.csv:":[207],"projects":[214,232,310,702,712,874,1127],"based":[216,311,322,875],"TLoC":[218,246,284,313,770,878],"number":[221,248,773,783,799,882],"commits":[223,250,289,775,801,884],"adr_template.csv:":[224],"which":[234,1044],"template":[236,326,862,914],"was":[237],"identified":[238],"project_stats.csv:":[243],"each":[252,638,758,777,852,1117],"Scripts":[254],"01_data_collection":[255],"01_clone_repos.sh:":[256],"Script":[257,267,281,294,328,345,355,369,379,395],"clone":[259,586],"locally":[260,1289],"repositories":[262,274,292,578,588,596],"02_create_codebase.sh:":[266],"merge":[269,695],"source":[270,697,719],"code":[271,698,720,785],"into":[275,703,739,866],"JSON":[278,704,742,1245],"03_project_stats.sh:":[280],"analyze":[283,1112],"amount":[287],"04_list_adr_files.py:":[293],"process":[296,446,1142],"list":[303],"files,":[305],"select":[306,319,831,906],"commits,":[315],"well":[317],"parsed":[325,861,915],"05_create_adr.sh:":[327],"move":[330],"projects'":[337],"codebases":[338],"02_llms":[343],"adr_judge.py:":[344],"using":[347,357,552,648,1159],"LRM":[348,367,408,1060,1081,1140,1287,1300,1350],"detect":[350],"violations":[353],"adr_validate.py:":[354],"3":[358,1160,1189,1352],"validate":[361,1163],"judgement":[364],"results":[365,405,423,431,438,451,535,1108,1136,1154,1187],"sbatch.sh:":[368],"submit":[371,1088,1171],"supercomputer":[376,1176],"03_data_analysis":[377],"rq1analysis.py:":[378],"calculate":[381,397,461,487],"overall":[383,463],"agreement":[384,464],"(Fleiss'":[385],"Kappa,":[386],"Agreement":[387,470,472],"by":[388,473,518,601,722,753],"category,":[389],"Jaccard":[392],"coefficients)":[393],"rq2analysis.py:":[394],"sklearn":[399,489,500],"metrics":[400,490],"Results":[401,411,504,512],"llms":[402],"judge_marco.csv:":[403],"Raw":[404,422,430,437],"(Marco-o1)":[409,1288,1351],"judge_marco_statuscorrection.csv:":[410],"through":[412,709],"status":[414,865,1114,1123,1129],"filtering":[415],"status_category.csv:":[416],"Status":[417],"extraction":[418],"mapping":[420],"validate_mistral.csv:":[421],"(validation)":[424,432,439],"Mistral":[427],"Nemo":[428],"validate_llama.csv:":[429],"Llama":[435],"validate_qwen.csv:":[436],"Qwen":[442,1357],"human_validation.csv:":[443],"Human":[444],"validation":[445,1186,1368],"sample":[449],"size":[450],"RQ1":[452],"agreement_analysis.csv:":[453],"Input":[454,480],"sent":[456,482],"rq1analysis.py":[459],"rq1data.csv:":[465],"Generated":[466,492],"(Overall":[468],"Inter-LLMs":[469],"Label":[474],"Class":[476],"Frequency)":[477],"RQ2":[478],"performance_analysis.csv:":[479],"rq2analysis.py":[485],"rq2data.csv:":[491],"(table":[494],"that":[495,911,1205],"present":[496],"needed":[499],"metrics)":[501],"RQ3":[502],"rq3.1data.csv:":[503],"category":[508],"(limitation":[509],"analysis)":[510,522],"rq3.2data.csv:":[511],"error":[515],"labels":[516],"(error":[520],"label":[521],"LLM_Input":[523,1041,1225],"Abelaa--pcmt:":[524],"One":[525,1241],"repo":[526],"example":[527,1036],"input":[531,1078,1223],"Replication":[532],"section":[537,1201],"steps":[540,554],"necessary":[541],"replicate":[543],"project,":[545,1118],"selection,":[547],"violation":[550],"detection":[551],"our":[556],"work.":[557],"Setting":[558],"up":[559,569],"Follow":[562],"INSTALL.md":[566],"set":[568,1234],"define":[573],"variables.":[575],"Cloning":[576,606],"Use":[579,690,763,821,946],"script":[581,692,707,765,780,823,836,948,959,1056,1301],"01_clone_repos.sh":[582],"attempt":[584],"dataset.":[591,939],"clones":[594],"directory":[599,671,751],"given":[600,752,812,889,964,1030],"$REPO_DIR":[603,977],"variable.":[605,816],"performed":[608],"over":[609,841],"https,":[610],"skipped":[615],"if":[616,627,726],"URL":[618,668],"found":[621,790,991],"(project":[622,631],"has":[623,632,1007],"been":[624,633],"deleted)":[625],"or":[626,1321],"authentication":[628],"requested":[630],"switched":[634],"private).":[636],"For":[637],"repository,":[639],"owner":[641],"name":[645,855],"concatenated":[647],"--":[649],"create":[651],"unique":[653],"string":[654],"identifying":[655],"be":[659,673,1029,1051,1370],"used":[660,678,1052],"directory,":[663],"i.e.":[664],"https://github.com/OWNER/REPO":[669],"OWNER--REPO.":[674],"convention":[676],"throughout":[680],"this":[681,1025],"scripts.":[685],"Creating":[686],"02_create_codebase.sh":[693],"cloned":[701,873,974],"goes":[708],"$REPO_DIR,":[715],"identifies":[716],"common":[723],"extensions,":[724],"checks":[725],"text,":[730],"non-empty":[731],"UTF-8":[734],"encoding,":[735],"merges":[737],"them":[738,918,970],"file.":[743],"written":[747],"$CODEBASE_DIR":[754,792,1006],"variable":[756],"project.":[759,778],"Analyzing":[760],"statistics":[762],"03_project_stats.sh":[766],"compute":[768],"count":[781],"lines":[786],"uses":[794],"git":[795],"check":[797],"$REPO_DIR.":[803],"saved":[807],"$STATS_FILE":[814],"Selecting":[817],"04_list_adr_files.py":[824],"parse":[826],"performs":[837],"following:":[839],"Iterate":[840],"ADR_dataset":[847],"write":[849],"information":[850],"owner,":[854],"URL,":[857],"file,":[859],"its":[860],"acceptance":[864],"adr_all.csv":[870],"Select":[871],"(4th":[879],"quartile)":[880],"(2nd":[885],"quartile":[886],"above)":[888],"project_stats.csv,":[891],"save":[893,917],"about":[895],"their":[897,913],"adr_selected_projects.csv":[904],"Further":[905],"have":[912,1208],"adr_template.csv":[923,926],"thus":[928],"contain":[929],"Adding":[940],"05_create_adr.sh":[949],"add":[951],"codebases.":[957],"takes":[960],"adr_template.csv,":[966],"attempts":[967],"corresponding":[973,982],"copy":[979],"$CODEBASE_DIR.":[985],"If":[986],"it":[992,1075],"skipped.":[994],"Final":[995],"selection":[997],"structure":[998,1026],"After":[999,1063,1150],"successfully":[1003],"executed,":[1004],"structure:":[1010,1098],"$CODEBASE_DIR/":[1011],"OWNER1--REPO1/":[1012],"OWNER1--REPO1--codebase.json":[1013],"ADR1.md":[1014,1019],"ADR2.md":[1015,1020],"...":[1016,1021,1239],"OWNER2--REPO2/":[1017],"OWNER2--REPO2--codebase.json":[1018],".../":[1022],"Files":[1023],"can":[1027,1050,1297,1369],"then":[1028],"LLMs.":[1032],"We":[1033,1083,1166],"provide":[1034],"one":[1035,1320,1362],"(one":[1037],"repo)":[1038],"folder,":[1042,1226],"Abelaa--pcmt,":[1043],"shows":[1045],"above":[1047,1067],"structure,":[1048,1072],"test":[1054],"input.":[1059],"Running":[1061,1141,1148],"(supercomputer)":[1062,1149],"described":[1071],"use":[1074,1084,1167],"running":[1080],"(Marco-o1).":[1082],"adr_judge.py":[1085],"sbatch.sh,":[1087,1170],"supercomputer,":[1093],"command":[1097,1180],"sbatch":[1099],"sbatch.sh":[1100],"\"MODEL_NAME\"":[1101],"PORT":[1102],"OUTPUT_FILE":[1103],"Then":[1104,1110,1182],"judge_marco.csv.":[1109],"situation":[1115],"try":[1119],"map":[1121],"remove":[1125],"whose":[1128],"\"rejected\".":[1131],"Finally,":[1132],"judge_marco_statuscorrection.csv.":[1137],"Now":[1138],"completed.":[1144],"Three":[1145],"Validation":[1147],"LRM,":[1156],"continue":[1158],"results.":[1165],"adr_validate.py":[1168],"same":[1179],"structure.":[1181],"LLMs:":[1190],"validate_mistral.csv,":[1191],"validate_llama.csv,":[1192],"validate_qwen.csv.":[1194],"Non-supercomputer":[1195],"(Local":[1196],"Demo,":[1197],"No":[1198],"SLURM)":[1199],"intended":[1203],"access":[1209],"supercomputer.":[1212],"It":[1213],"runs":[1214],"minimal":[1216],"demo":[1217,1229],"machine.":[1221],"Demo":[1222],"Abelaa--pcmt":[1227],"includes:":[1231],"A":[1232,1316],"small":[1233],"ADRs":[1236],"(e.g.,":[1237,1247],"adr-001.md":[1238],"adr-007.md)":[1240],"snapshot":[1243],"format":[1246],"Abelaa--pcmt--codebase.json)":[1248],"Start":[1249],"an":[1250,1258],"OpenAI-compatible":[1251,1259],"endpoint":[1253],"(vLLM)":[1254],"expects":[1257],"/v1/chat/completions":[1260],"endpoint.":[1261],"Example:":[1262],"start":[1263],"vLLM":[1266,1292,1360,1381],"server":[1267,1293,1365],"(single":[1268],"GPU)":[1269],"python":[1270,1303],"-m":[1271],"vllm.entrypoints.openai.api_server":[1272],"\\":[1273,1276,1279,1305,1308,1311],"--model":[1274],"AIDC-AI/Marco-o1":[1275],"--port":[1277],"8000":[1278],"--tensor-parallel-size":[1280],"1":[1281],"Sanity":[1282],"check:":[1283],"curl":[1284],"http://localhost:8000/v1/models":[1285],"Run":[1286],"Once":[1290],"running,":[1295],"you":[1296],"using:":[1302],"Scripts/02_llms/adr_judge.py":[1304],"--model_name":[1306],"\"AIDC-AI/Marco-o1\"":[1307],"--api_url":[1309],"\"http://localhost:8000/v1/chat/completions\"":[1310],"--output_file":[1312],"Results/llms/judge_marco_demo.csv":[1313],"Expected":[1314],"output:":[1315],"containing":[1319],"more":[1322],"rows":[1323],"per":[1324,1364],"extracted":[1325],"rule.":[1326],"output":[1328],"should":[1329],"include":[1330],"at":[1331],"least":[1332],"fields:":[1334],"Rule,":[1335],"Judgement,":[1336],"Reason.":[1338],"Notes":[1339],"multi-model":[1341,1367],"setup":[1342],"(4":[1343],"models)":[1344],"paper":[1346],"evaluates":[1347],"4":[1348],"models:":[1349],"validator":[1353],"models":[1354],"(Mistral":[1355],"/":[1356,1358],"Llama)":[1359],"serves":[1361],"model":[1363],"instance.Therefore,":[1366],"done":[1371],"two":[1373],"ways:":[1374],"Sequential":[1375],"mode":[1376],"(recommended":[1377],"demo):":[1379],"stop":[1380],"re":[1383]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-02-13T00:00:00"}