{"id":"https://openalex.org/W3204368535","doi":"https://doi.org/10.1109/tse.2021.3113558","title":"Use and Misuse of the Term \u201cExperiment\u201d in Mining Software Repositories Research","display_name":"Use and Misuse of the Term \u201cExperiment\u201d in Mining Software Repositories Research","publication_year":2021,"publication_date":"2021-09-24","ids":{"openalex":"https://openalex.org/W3204368535","doi":"https://doi.org/10.1109/tse.2021.3113558","mag":"3204368535"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2021.3113558","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2021.3113558","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2110.09366","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076035932","display_name":"Claudia Ayala","orcid":"https://orcid.org/0000-0002-6262-3698"},"institutions":[{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Claudia Ayala","raw_affiliation_strings":["Universitat Politècnica de Catalunya, BarcelonaTECH, Barcelona, Spain"],"raw_orcid":"https://orcid.org/0000-0002-6262-3698","affiliations":[{"raw_affiliation_string":"Universitat Politècnica de Catalunya, BarcelonaTECH, Barcelona, Spain","institution_ids":["https://openalex.org/I9617848"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044710601","display_name":"Burak Turhan","orcid":"https://orcid.org/0000-0003-1511-2163"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]},{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["AU","FI"],"is_corresponding":false,"raw_author_name":"Burak Turhan","raw_affiliation_strings":["University of Oulu, Oulu, Finland","Monash University, Clayton, VIC, Australia"],"raw_orcid":"https://orcid.org/0000-0003-1511-2163","affiliations":[{"raw_affiliation_string":"University of Oulu, Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]},{"raw_affiliation_string":"Monash University, Clayton, VIC, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027686521","display_name":"Xavier Franch","orcid":"https://orcid.org/0000-0001-9733-8830"},"institutions":[{"id":"https://openalex.org/I9617848","display_name":"Universitat Polit\u00e8cnica de Catalunya","ror":"https://ror.org/03mb6wj31","country_code":"ES","type":"education","lineage":["https://openalex.org/I9617848"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Xavier Franch","raw_affiliation_strings":["Universitat Politècnica de Catalunya, BarcelonaTECH, Barcelona, Spain"],"raw_orcid":"https://orcid.org/0000-0001-9733-8830","affiliations":[{"raw_affiliation_string":"Universitat Politècnica de Catalunya, BarcelonaTECH, Barcelona, Spain","institution_ids":["https://openalex.org/I9617848"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056117323","display_name":"Nat\u00e1lia Juristo","orcid":"https://orcid.org/0000-0002-2465-7141"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Natalia Juristo","raw_affiliation_strings":["Universidad Politécnica de Madrid, Boadilla del Monte, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universidad Politécnica de Madrid, Boadilla del Monte, Spain","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.1031,"has_fulltext":false,"cited_by_count":22,"citation_normalized_percentile":{"value":0.9549082,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"48","issue":"11","first_page":"4229","last_page":"4248"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9779000282287598,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/term","display_name":"Term (time)","score":0.7386797070503235},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.7132420539855957},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6993380188941956},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.6583617329597473},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.525093674659729},{"id":"https://openalex.org/keywords/observational-study","display_name":"Observational study","score":0.5158504843711853},{"id":"https://openalex.org/keywords/empirical-evidence","display_name":"Empirical evidence","score":0.4860003888607025},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.477597713470459},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.4311411380767822},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.18835866451263428},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.15315744280815125}],"concepts":[{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.7386797070503235},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.7132420539855957},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6993380188941956},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.6583617329597473},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.525093674659729},{"id":"https://openalex.org/C23131810","wikidata":"https://www.wikidata.org/wiki/Q818574","display_name":"Observational study","level":2,"score":0.5158504843711853},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.4860003888607025},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.477597713470459},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.4311411380767822},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18835866451263428},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.15315744280815125},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tse.2021.3113558","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2021.3113558","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2110.09366","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.09366","pdf_url":"https://arxiv.org/pdf/2110.09366","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:oulu.fi:nbnfi-fe2021100649447","is_oa":false,"landing_page_url":"http://urn.fi/urn:nbn:fi-fe2021100649447","pdf_url":null,"source":{"id":"https://openalex.org/S4306400284","display_name":"University of Oulu Repository (University of Oulu)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98381234","host_organization_name":"University of Oulu","host_organization_lineage":["https://openalex.org/I98381234"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:upcommons.upc.edu:2117/363040","is_oa":true,"landing_page_url":"http://hdl.handle.net/2117/363040","pdf_url":null,"source":{"id":"https://openalex.org/S4377196262","display_name":"UPCommons institutional repository (Universitat Polit\u00e8cnica de Catalunya)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I9617848","host_organization_name":"Universitat Polit\u00e8cnica de Catalunya","host_organization_lineage":["https://openalex.org/I9617848"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2110.09366","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.09366","pdf_url":"https://arxiv.org/pdf/2110.09366","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.5199999809265137}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":76,"referenced_works":["https://openalex.org/W88066535","https://openalex.org/W611321302","https://openalex.org/W1511267087","https://openalex.org/W1522729716","https://openalex.org/W1570448133","https://openalex.org/W1577119245","https://openalex.org/W1583096133","https://openalex.org/W1746353556","https://openalex.org/W1914969610","https://openalex.org/W1965772655","https://openalex.org/W1968451194","https://openalex.org/W1989818349","https://openalex.org/W1994922758","https://openalex.org/W1999798506","https://openalex.org/W2008164297","https://openalex.org/W2009223100","https://openalex.org/W2011210303","https://openalex.org/W2030857403","https://openalex.org/W2031107335","https://openalex.org/W2049158239","https://openalex.org/W2054790703","https://openalex.org/W2058834052","https://openalex.org/W2084429940","https://openalex.org/W2086702565","https://openalex.org/W2090908431","https://openalex.org/W2094419105","https://openalex.org/W2105033346","https://openalex.org/W2105527306","https://openalex.org/W2109076419","https://openalex.org/W2110307645","https://openalex.org/W2111199933","https://openalex.org/W2118328848","https://openalex.org/W2126130217","https://openalex.org/W2127546927","https://openalex.org/W2138163761","https://openalex.org/W2138428785","https://openalex.org/W2140190241","https://openalex.org/W2162739315","https://openalex.org/W2163851162","https://openalex.org/W2169090130","https://openalex.org/W2169843470","https://openalex.org/W2194085927","https://openalex.org/W2275737843","https://openalex.org/W2400138769","https://openalex.org/W2486444208","https://openalex.org/W2505184898","https://openalex.org/W2544786282","https://openalex.org/W2562248917","https://openalex.org/W2593710147","https://openalex.org/W2621604851","https://openalex.org/W2772583926","https://openalex.org/W2811455890","https://openalex.org/W2914024328","https://openalex.org/W2938140497","https://openalex.org/W2954093663","https://openalex.org/W2963203256","https://openalex.org/W2983928456","https://openalex.org/W3023468469","https://openalex.org/W3030475425","https://openalex.org/W3081764108","https://openalex.org/W3141588637","https://openalex.org/W3144106047","https://openalex.org/W3162439866","https://openalex.org/W4237578926","https://openalex.org/W4237617498","https://openalex.org/W4240065799","https://openalex.org/W4245192846","https://openalex.org/W4287867080","https://openalex.org/W4291213652","https://openalex.org/W6631116052","https://openalex.org/W6631841644","https://openalex.org/W6634970337","https://openalex.org/W6729242656","https://openalex.org/W6739189794","https://openalex.org/W6752670219","https://openalex.org/W6774519044"],"related_works":["https://openalex.org/W4381328000","https://openalex.org/W2109147503","https://openalex.org/W2103779230","https://openalex.org/W2911339178","https://openalex.org/W334096847","https://openalex.org/W2322581019","https://openalex.org/W3141050733","https://openalex.org/W976271273","https://openalex.org/W4281748496","https://openalex.org/W2800302421"],"abstract_inverted_index":{"The":[0],"significant":[1],"momentum":[2],"and":[3,18,63,88,92,196],"importance":[4],"of":[5,43,50,60,85,94,113,123,155,160,187,190,202],"Mining":[6],"Software":[7,11],"Repositories":[8],"(MSR)":[9],"in":[10,70,96,117,147],"Engineering":[12],"(SE)":[13],"has":[14],"fostered":[15],"new":[16],"opportunities":[17],"challenges":[19],"for":[20,172],"extensive":[21],"empirical":[22,33,40,86],"research.":[23],"However,":[24],"MSR":[25,51,61,100,177,203],"researchers":[26,178],"seem":[27],"to":[28,30,109,127,163,180,198],"struggle":[29],"characterize":[31],"the":[32,38,48,57,76,90,111,114,124,145,152,156,169,185,188,200],"methods":[34],"they":[35,143],"use":[36,112,144],"into":[37],"existing":[39],"SE":[41,71],"body":[42],"knowledge.":[44],"This":[45],"is":[46],"especially":[47],"case":[49],"experiments.":[52,204],"To":[53],"provide":[54,194],"evidence":[55],"on":[56],"special":[58],"characteristics":[59],"experiments":[62,67,95,173],"their":[64,191],"differences":[65],"with":[66,174],"traditionally":[68],"acknowledged":[69],"so":[72,142],"far,":[73],"we":[74],"elicited":[75],"hallmarks":[77,91],"that":[78,121],"differentiate":[79],"an":[80,129,134],"experiment":[81,116,130,135,167],"from":[82,103],"other":[83],"types":[84,93],"studies":[87],"characterized":[89],"MSR.":[97,118],"We":[98,119,193],"analyzed":[99],"literature":[101],"obtained":[102],"a":[104,148,164],"small-scale":[105],"systematic":[106],"mapping":[107],"study":[108],"assess":[110],"term":[115,146],"found":[120],"19%":[122],"papers":[125],"claiming":[126],"be":[128],"are":[131],"indeed":[132],"not":[133],"at":[136],"all":[137],"but":[138],"also":[139],"observational":[140],"studies,":[141],"misleading":[149],"way.":[150],"From":[151],"remaining":[153],"81%":[154],"papers,":[157],"only":[158],"one":[159],"them":[161],"refers":[162],"genuine":[165],"controlled":[166],"while":[168],"others":[170],"stand":[171],"limited":[175],"control.":[176],"tend":[179],"overlook":[181],"such":[182],"limitations,":[183],"compromising":[184],"interpretation":[186],"results":[189],"studies.":[192],"recommendations":[195],"insights":[197],"support":[199],"improvement":[201]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":1}],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2021-10-11T00:00:00"}