{"id":"https://openalex.org/W7108334604","doi":"https://doi.org/10.5281/zenodo.17781419","title":"A Critical Reflection on the State of Data Analysis in Empirical Software Engineering - Replication Package (TOSEM Journal)","display_name":"A Critical Reflection on the State of Data Analysis in Empirical Software Engineering - Replication Package (TOSEM Journal)","publication_year":2025,"publication_date":"2025-12-01","ids":{"openalex":"https://openalex.org/W7108334604","doi":"https://doi.org/10.5281/zenodo.17781419"},"language":null,"primary_location":{"id":"doi:10.5281/zenodo.17781419","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.17781419","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},"type":"software","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.17781419","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Esposito, Matteo","orcid":"https://orcid.org/0000-0002-8451-3668"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Esposito, Matteo","raw_affiliation_strings":["University of Oulu"],"raw_orcid":"https://orcid.org/0000-0002-8451-3668","affiliations":[{"raw_affiliation_string":"University of Oulu","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Robredo, Mikel","orcid":"https://orcid.org/0009-0001-9870-1504"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Robredo, Mikel","raw_affiliation_strings":["University of Oulu"],"raw_orcid":"https://orcid.org/0009-0001-9870-1504","affiliations":[{"raw_affiliation_string":"University of Oulu","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Sridharan, Murali","orcid":"https://orcid.org/0000-0002-5212-588X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sridharan, Murali","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-5212-588X","affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Travassos, Guilherme Horta","orcid":"https://orcid.org/0000-0002-4258-0424"},"institutions":[{"id":"https://openalex.org/I122140584","display_name":"Universidade Federal do Rio de Janeiro","ror":"https://ror.org/03490as77","country_code":"BR","type":"education","lineage":["https://openalex.org/I122140584"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Travassos, Guilherme Horta","raw_affiliation_strings":["Universidade Federal do Rio de Janeiro"],"raw_orcid":"https://orcid.org/0000-0002-4258-0424","affiliations":[{"raw_affiliation_string":"Universidade Federal do Rio de Janeiro","institution_ids":["https://openalex.org/I122140584"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Pe\u00f1aloza, Rafael","orcid":"https://orcid.org/0000-0002-2693-5790"},"institutions":[{"id":"https://openalex.org/I66752286","display_name":"University of Milano-Bicocca","ror":"https://ror.org/01ynf4891","country_code":"IT","type":"education","lineage":["https://openalex.org/I66752286"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Pe\u00f1aloza, Rafael","raw_affiliation_strings":["University of Milano-Bicocca"],"raw_orcid":"https://orcid.org/0000-0002-2693-5790","affiliations":[{"raw_affiliation_string":"University of Milano-Bicocca","institution_ids":["https://openalex.org/I66752286"]}]},{"author_position":"last","author":{"id":null,"display_name":"Lenarduzzi, Valentina","orcid":"https://orcid.org/0000-0003-0511-5133"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Lenarduzzi, Valentina","raw_affiliation_strings":["University of Oulu"],"raw_orcid":"https://orcid.org/0000-0003-0511-5133","affiliations":[{"raw_affiliation_string":"University of Oulu","institution_ids":["https://openalex.org/I98381234"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.7080000042915344},{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.6733999848365784},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.5734000205993652},{"id":"https://openalex.org/keywords/raw-data","display_name":"Raw data","score":0.5715000033378601},{"id":"https://openalex.org/keywords/replication","display_name":"Replication (statistics)","score":0.5454000234603882},{"id":"https://openalex.org/keywords/data-file","display_name":"Data file","score":0.527999997138977},{"id":"https://openalex.org/keywords/file-format","display_name":"File format","score":0.503600001335144},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4991999864578247},{"id":"https://openalex.org/keywords/timeline","display_name":"Timeline","score":0.42089998722076416}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7990000247955322},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.7080000042915344},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.6733999848365784},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.5734000205993652},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.5715000033378601},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.5454000234603882},{"id":"https://openalex.org/C171730128","wikidata":"https://www.wikidata.org/wiki/Q5227290","display_name":"Data file","level":2,"score":0.527999997138977},{"id":"https://openalex.org/C97250363","wikidata":"https://www.wikidata.org/wiki/Q235557","display_name":"File format","level":2,"score":0.503600001335144},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4991999864578247},{"id":"https://openalex.org/C4438859","wikidata":"https://www.wikidata.org/wiki/Q186117","display_name":"Timeline","level":2,"score":0.42089998722076416},{"id":"https://openalex.org/C88006597","wikidata":"https://www.wikidata.org/wiki/Q690117","display_name":"Disk formatting","level":2,"score":0.3968000113964081},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.38269999623298645},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.37599998712539673},{"id":"https://openalex.org/C30088001","wikidata":"https://www.wikidata.org/wiki/Q102014","display_name":"Hyperlink","level":3,"score":0.36550000309944153},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35100001096725464},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.3440999984741211},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3411000072956085},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3370000123977661},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.3215999901294708},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2980000078678131},{"id":"https://openalex.org/C65377053","wikidata":"https://www.wikidata.org/wiki/Q1572121","display_name":"Image file formats","level":3,"score":0.28110000491142273},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.27480000257492065},{"id":"https://openalex.org/C198140048","wikidata":"https://www.wikidata.org/wiki/Q10859422","display_name":"Software versioning","level":3,"score":0.2718999981880188},{"id":"https://openalex.org/C37789001","wikidata":"https://www.wikidata.org/wiki/Q782543","display_name":"Graphical user interface","level":2,"score":0.2655999958515167},{"id":"https://openalex.org/C172367668","wikidata":"https://www.wikidata.org/wiki/Q6504956","display_name":"Data visualization","level":3,"score":0.2578999996185303},{"id":"https://openalex.org/C3019730874","wikidata":"https://www.wikidata.org/wiki/Q11272","display_name":"Microsoft excel","level":2,"score":0.2572000026702881},{"id":"https://openalex.org/C8797682","wikidata":"https://www.wikidata.org/wiki/Q2115","display_name":"XML","level":2,"score":0.2547999918460846},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.2547000050544739}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.5281/zenodo.17781419","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.17781419","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Software"}],"best_oa_location":{"id":"doi:10.5281/zenodo.17781419","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.17781419","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"This":[0,81,535],"is":[1,462,538],"a":[2],"replication":[3,65,82,536],"package":[4,83,537],"and":[5,93,143,174,179,205,242,295,364,370,393,511,529,545],"online":[6],"appendix":[7],"for":[8,104,191,198,221,299,368,397,416,455,564,567],"the":[9,15,26,37,43,46,57,75,90,98,107,113,123,181,189,230,275,300,306,330,334,340,347,351,361,383,387,398,473,489,495,501,550],"paper:":[10],"\"A":[11],"Critical":[12],"Reflection":[13],"on":[14],"State":[16],"of":[17,77,106,129,141,207,305,354,421],"Data":[18],"Analysis":[19],"in":[20,97,339,350,497,516,532],"Empirical":[21],"Software":[22],"Engineering\".":[23],"NOTE:":[24],"All":[25,459,468,521],"file":[27,220],"hyperlinks":[28],"included":[29,338],"into":[30],"this":[31,64,78,355],"README":[32],"will":[33],"not":[34,55],"work":[35],"within":[36,63,386,549],"Zenodo":[38],"description,":[39],"but":[40],"only":[41],"once":[42],"reader":[44],"downloads":[45],"registration":[47],"package.":[48,66],"Due":[49],"to":[50,73,88,177,483,499,540,558],"data-management":[51],"ethics,":[52],"we":[53],"do":[54],"release":[56],"raw":[58,236],"corpus":[59],"stracted":[60],"from":[61,188,274,329,360],"Scopus":[62],"For":[67],"more":[68],"information":[69],"o2e":[70],"feel":[71],"free":[72],"contact":[74],"authors":[76,556],"paper.":[79,99,124],"Contents":[80,359],"contains":[84],"all":[85,222,346,412,453],"materials":[86],"necessary":[87],"reproduce":[89],"analyses,":[91],"results,":[92],"methodological":[94,157,547],"assessments":[95],"presented":[96],"INSTALL:":[100],"Detailed":[101],"installation":[102],"instructions":[103],"each":[105,192],"used":[108,121,176,367,415],"tools":[109],"as":[110,112],"well":[111],"required":[114],"Python":[115],"dependencies.":[116],"1.":[117,491],"Figures/":[118],"Graphical":[119],"material":[120],"throughout":[122],"global_not_adequate_strategies.pdf":[125],"\u2014":[126,134,139,147,153,161,166,185,195,203,211,217],"Global":[127,284],"distribution":[128,432],"inadequate":[130],"analysis":[131,290,457],"strategies.":[132],"not_adequate_strategies.pdf":[133],"Strategy-level":[135],"inadequacy":[136],"illustration.":[137],"RW-chronos.pdf":[138],"Timeline":[140],"replications":[142],"research":[144],"waves.":[145],"study-design.pdf":[146],"Study":[148,490],"design":[149],"overview.":[150],"BPM_study_setup.pdf,":[151],"BPM-Vertical.pdf":[152],"BPM":[154],"diagrams":[155],"supporting":[156],"interpretation.":[158],"Pattern.pdf,":[159],"Pattern_small.pdf":[160],"Coding":[162,505],"pattern":[163],"visualizations.":[164],"EntireProcess_.pdf":[165],"End-to-end":[167],"process":[168],"depiction.":[169],"2.":[170,504],"metadata/":[171,498],"Main":[172],"scripts":[173,469,496],"data":[175,399,426,461],"retrieve":[178],"curate":[180],"literature":[182],"corpus.":[183,332,503],"fetch_by_year.py":[184],"Retrieves":[186],"metadata":[187,200,219,331],"API":[190],"year.":[193],"retrieve_articles.batch":[194],"Batch":[196],"script":[197],"large-scale":[199],"retrieval.":[201],"filter_se_articles.py":[202],"Post-processing":[204],"filtering":[206],"non-SE":[208],"publications.":[209],"se_journals_conferences.py":[210],"Canonical":[212],"SE":[213,336,394],"venue":[214,395],"list.":[215],"main_metadata_1993_2023.json":[216],"Final":[218],"candidate":[223],"articles":[224,337],"across":[225],"1993\u20132023.":[226],"3.":[227,518],"manual_annotations/":[228],"Contains":[229,333,345],"full":[231],"manual":[232,246,258],"validation":[233],"process,":[234],"including":[235],"annotations,":[237,239],"synthetized":[238],"cleaned":[240],"data,":[241],"statistical":[243,301,431,522],"summaries.":[244],"Raw":[245,249],"annotations:":[247,259],"DataForManualAnalysis.xlsx:":[248],"stratified":[250,276],"sample":[251,255],"papers":[252,324,348],"metadata.":[253],"Stratified":[254],"sheets":[256],"with":[257,278,382],"(S1.xlsx,":[260],"S2.xlsx,":[261],"\u2026)":[262],"\u2192":[263],"e.g.,":[264,281],"S1.xlsx":[265],"annotations_with_statistics/":[266],"Automatically":[267],"computed":[268],"coding":[269],"statistics:":[270],"S*_statistics.xlsx:":[271],"Replicated":[272],"files":[273,291,319,531],"samples":[277],"accuracy":[279,285],"metrics":[280],"S1_statistics.xlsx":[282],"global_stats.xlsx:":[283],"metrics.":[286],"4.":[287],"HT_jmp/":[288],"Statistical":[289],"(JMP":[292],".jrp),":[293],"PDFs,":[294],"post-hoc":[296,441],"comparisons":[297],"performed":[298],"hypothesis":[302,436],"testing":[303,437],"stage":[304],"study.":[307,356],"good_normality_test.jrp,":[308],"bad_normality_test.jrp":[309],"ht_good_category.jrp,":[310],"ht_bad_category.jrp":[311],"ht_good_category.pdf,":[312],"ht_bad_category.pdf":[313],"post_hoc_good_results.xlsx,":[314],"post_hoc_bad_results.xlsx":[315],"5.":[316],"retracted_papers/":[317],"Supporting":[318],"identifying":[320],"retracted":[321,335],"or":[322,478],"withdrawn":[323],"that":[325],"must":[326],"be":[327,525],"excluded":[328],"expanded":[341],"Appendix.":[342],"6.":[343],"RWs/":[344],"considered":[349,384],"Related":[352],"Works":[353],"8.":[357],"Workshop_Data/":[358],"pilot":[362],"study":[363],"focus":[365],"groups":[366],"RQ1":[369],"RQ2.":[371],"Pilot":[372],"Study/":[373],"PilotStudy.xlsx":[374],"Focus":[375],"Group/":[376],"focus_group_answers.xlsx":[377],"9.":[378],"Further":[379],"content":[380,480],"Tables":[381],"taxonomies":[385],"study,":[388],"their":[389,484],"LaTeX":[390],"table":[391],"version,":[392],"lists":[396],"collection.":[400],"taxonomy-keywords.csv":[401],"taxonomy_table.tex":[402],"taxonomy_longtable.tex":[403],"SE-venues.csv":[404],"SE-venues.tex":[405],"10.":[406],"text_classification_scripts/":[407],"Script":[408],"Purpose":[409],"keywords.py":[410],"Stores":[411],"keyword":[413],"sets":[414],"classification":[417],"load_model.py":[418],"Detects":[419,425,430,435,440],"mentions":[420],"statistical/ML":[422],"models":[423],"load_pre_processing.py":[424],"pre-processing":[427],"procedures":[428],"load_distribution_check.py":[429],"tests":[433,445],"load_hypothesis_testing.py":[434],"text":[438],"load_posthoc.py":[439],"analyses":[442,523],"after":[443],"significance":[444],"load_performance.py":[446],"Extracts":[447],"model":[448],"performance":[449],"reporting":[450],"section_modeling.py":[451],"Orchestrates":[452],"classifiers":[454],"full-section":[456],"License":[458],"generated":[460],"released":[463,471],"under":[464,472],"CC":[465],"BY":[466],"4.0.":[467],"are":[470,481,514],"MIT":[474],"License.":[475],"External":[476],"datasets":[477,510],"API-retrieved":[479],"subject":[482],"providers\u2019":[485],"license":[486],"terms.":[487],"Reproducing":[488],"Metadata":[492],"Retrieval":[493],"Run":[494],"generate":[500],"article":[502],"Consolidation":[506],"&":[507],"Statistics":[508],"Cleaned":[509],"annotation":[512],"statistics":[513],"located":[515],"manual_annotations/.":[517],"Hypothesis":[519],"Testing":[520],"can":[524],"replicated":[526],"using":[527],".jrp":[528],".xlsx":[530],"HT_jmp/.":[533],"Acknowledgements":[534],"provided":[539],"promote":[541],"transparency,":[542],"facilitate":[543],"reuse,":[544],"support":[546],"reflection":[548],"empirical":[551],"software":[552],"engineering":[553],"community.":[554],"The":[555],"wish":[557],"acknowledge":[559],"CSC":[560],"\u2013":[561],"IT":[562],"Center":[563],"Science,":[565],"Finland,":[566],"computational":[568],"resources.":[569]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-12-03T00:00:00"}