{"id":"https://openalex.org/W4403413367","doi":"https://doi.org/10.1145/3674805.3686691","title":"Threats to Validity in Software Engineering \u2013 hypocritical paper section or essential analysis?","display_name":"Threats to Validity in Software Engineering \u2013 hypocritical paper section or essential analysis?","publication_year":2024,"publication_date":"2024-10-15","ids":{"openalex":"https://openalex.org/W4403413367","doi":"https://doi.org/10.1145/3674805.3686691"},"language":"en","primary_location":{"id":"doi:10.1145/3674805.3686691","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3674805.3686691","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th ACM/IEEE International Symposium on Empirical Software Engineering and Measurement","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3674805.3686691","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088101716","display_name":"Patricia Lago","orcid":"https://orcid.org/0000-0002-2234-0845"},"institutions":[{"id":"https://openalex.org/I865915315","display_name":"Vrije Universiteit Amsterdam","ror":"https://ror.org/008xxew50","country_code":"NL","type":"education","lineage":["https://openalex.org/I865915315"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Patricia Lago","raw_affiliation_strings":["Software and Sustainability Research Group, Vrije Universiteit Amsterdam, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-2234-0845","affiliations":[{"raw_affiliation_string":"Software and Sustainability Research Group, Vrije Universiteit Amsterdam, Netherlands","institution_ids":["https://openalex.org/I865915315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087032437","display_name":"Per Runeson","orcid":"https://orcid.org/0000-0003-2795-4851"},"institutions":[{"id":"https://openalex.org/I187531555","display_name":"Lund University","ror":"https://ror.org/012a77v79","country_code":"SE","type":"education","lineage":["https://openalex.org/I187531555"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Per Runeson","raw_affiliation_strings":["Lund University, Sweden"],"raw_orcid":"https://orcid.org/0000-0003-2795-4851","affiliations":[{"raw_affiliation_string":"Lund University, Sweden","institution_ids":["https://openalex.org/I187531555"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004286923","display_name":"Qunying Song","orcid":"https://orcid.org/0000-0002-8653-0250"},"institutions":[{"id":"https://openalex.org/I187531555","display_name":"Lund University","ror":"https://ror.org/012a77v79","country_code":"SE","type":"education","lineage":["https://openalex.org/I187531555"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Qunying Song","raw_affiliation_strings":["Lund University, Sweden"],"raw_orcid":"https://orcid.org/0000-0002-8653-0250","affiliations":[{"raw_affiliation_string":"Lund University, Sweden","institution_ids":["https://openalex.org/I187531555"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5041332104","display_name":"Roberto Verdecchia","orcid":"https://orcid.org/0000-0001-9206-6637"},"institutions":[{"id":"https://openalex.org/I45084792","display_name":"University of Florence","ror":"https://ror.org/04jr1s763","country_code":"IT","type":"education","lineage":["https://openalex.org/I45084792"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Roberto Verdecchia","raw_affiliation_strings":["University of Florence, Italy"],"raw_orcid":"https://orcid.org/0000-0001-9206-6637","affiliations":[{"raw_affiliation_string":"University of Florence, Italy","institution_ids":["https://openalex.org/I45084792"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"314","last_page":"324"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9767000079154968,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/section","display_name":"Section (typography)","score":0.6389210820198059},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5336775183677673},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4117595851421356},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3899089992046356},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.3332652449607849},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.27507156133651733},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.13849669694900513},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.0900304913520813}],"concepts":[{"id":"https://openalex.org/C2780129039","wikidata":"https://www.wikidata.org/wiki/Q1931107","display_name":"Section (typography)","level":2,"score":0.6389210820198059},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5336775183677673},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4117595851421356},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3899089992046356},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.3332652449607849},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.27507156133651733},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.13849669694900513},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0900304913520813}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1145/3674805.3686691","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3674805.3686691","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th ACM/IEEE International Symposium on Empirical Software Engineering and Measurement","raw_type":"proceedings-article"},{"id":"pmh:oai:research.vu.nl:publications/c3eddc62-4d2a-466d-832a-9f427d6ad9d8","is_oa":true,"landing_page_url":"https://research.vu.nl/en/publications/c3eddc62-4d2a-466d-832a-9f427d6ad9d8","pdf_url":null,"source":{"id":"https://openalex.org/S4306401107","display_name":"VU Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I865915315","host_organization_name":"Vrije Universiteit Amsterdam","host_organization_lineage":["https://openalex.org/I865915315"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Lago, P, Runeson, P, Song, Q & Verdecchia, R 2024, Threats to Validity in Software Engineering - hypocritical paper section or essential analysis? in X Franch & M Daneva (eds), ESEM '24: Proceedings of the 18th ACM/IEEE International Symposium on Empirical Software Engineering and Measurement. International Symposium on Empirical Software Engineering and Measurement, IEEE Computer Society, pp. 314-324, 18th ACM/IEEE International Symposium on Empirical Software Engineering and Measurement, ESEM 2024, Barcelona, Spain, 24/10/24. https://doi.org/10.1145/3674805.3686691","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:flore.unifi.it:2158/1400258","is_oa":false,"landing_page_url":"https://hdl.handle.net/2158/1400258","pdf_url":null,"source":{"id":"https://openalex.org/S4306402033","display_name":"Florence Research (University of Florence)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I45084792","host_organization_name":"University of Florence","host_organization_lineage":["https://openalex.org/I45084792"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:lup.lub.lu.se:cdaa9e06-7308-42bc-aee4-6875a94f0023","is_oa":false,"landing_page_url":"https://lup.lub.lu.se/record/cdaa9e06-7308-42bc-aee4-6875a94f0023","pdf_url":null,"source":{"id":"https://openalex.org/S4306400536","display_name":"Lund University Publications (Lund University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I187531555","host_organization_name":"Lund University","host_organization_lineage":["https://openalex.org/I187531555"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferencePaper"},{"id":"pmh:oai:research.vu.nl:openaire/c3eddc62-4d2a-466d-832a-9f427d6ad9d8","is_oa":true,"landing_page_url":"https://hdl.handle.net/1871.1/c3eddc62-4d2a-466d-832a-9f427d6ad9d8","pdf_url":null,"source":{"id":"https://openalex.org/S4306401107","display_name":"VU Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I865915315","host_organization_name":"Vrije Universiteit Amsterdam","host_organization_lineage":["https://openalex.org/I865915315"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Lago, P, Runeson, P, Song, Q & Verdecchia, R 2024, Threats to Validity in Software Engineering - hypocritical paper section or essential analysis? in X Franch & M Daneva (eds), ESEM '24: Proceedings of the 18th ACM/IEEE International Symposium on Empirical Software Engineering and Measurement. International Symposium on Empirical Software Engineering and Measurement, IEEE Computer Society, pp. 314-324, 18th ACM/IEEE International Symposium on Empirical Software Engineering and Measurement, ESEM 2024, Barcelona, Spain, 24/10/24. https://doi.org/10.1145/3674805.3686691","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.1145/3674805.3686691","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3674805.3686691","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th ACM/IEEE International Symposium on Empirical Software Engineering and Measurement","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W240302486","https://openalex.org/W1499730430","https://openalex.org/W1575714269","https://openalex.org/W1876404477","https://openalex.org/W2088411581","https://openalex.org/W2094419105","https://openalex.org/W2097850216","https://openalex.org/W2123470003","https://openalex.org/W2809423099","https://openalex.org/W2890801208","https://openalex.org/W2891421390","https://openalex.org/W2897889469","https://openalex.org/W2959088703","https://openalex.org/W3030475425","https://openalex.org/W3081764108","https://openalex.org/W3163740924","https://openalex.org/W4233152543","https://openalex.org/W4239863959","https://openalex.org/W4241437391","https://openalex.org/W4285247202","https://openalex.org/W4285248702","https://openalex.org/W4291213652","https://openalex.org/W4386475806","https://openalex.org/W4386584974","https://openalex.org/W4391832619","https://openalex.org/W4392347631","https://openalex.org/W6911656332"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2560853036","https://openalex.org/W2566696415","https://openalex.org/W1563787543","https://openalex.org/W4239980664","https://openalex.org/W4249026152","https://openalex.org/W4231213805","https://openalex.org/W4244585678","https://openalex.org/W2909241626"],"abstract_inverted_index":{"Background:":[0],"In":[1],"recent":[2],"years,":[3],"a":[4,50,155],"discourse":[5],"on":[6,69],"how":[7],"to":[8,14,17,32,40,84,101,111,145,158],"systematically":[9,33],"consider":[10,146],"and":[11,75,94,117,134,150,152],"report":[12],"threats":[13,39,83,103],"validity":[15,41],"started":[16],"gain":[18],"momentum":[19],"within":[20],"the":[21,35,58,65,102,107,131,139,160],"empirical":[22,166],"software":[23,44,167],"engineering":[24,45,168],"community.":[25],"Aims:":[26],"With":[27],"this":[28],"study,":[29],"we":[30,142],"aim":[31],"underpin":[34],"current":[36,99],"state":[37,105],"of":[38,82,92,106,130,162],"practices":[42],"in":[43,115,165],"research.":[46,169],"Method:":[47],"We":[48],"conduct":[49],"literature":[51],"review":[52],"comprising":[53],"91":[54],"papers":[55],"awarded":[56],"with":[57],"ACM":[59],"SIGSOFT":[60],"Distinguished":[61],"Paper":[62],"Award":[63],"at":[64],"ACM/IEEE":[66],"International":[67],"Conference":[68],"Software":[70],"Engineering.":[71],"Data":[72],"is":[73],"extracted":[74],"analyzed":[76],"by":[77],"considering":[78],"six":[79],"main":[80],"facets":[81],"validity,":[85],"e.g.,":[86],"their":[87],"explicit":[88],"documentation,":[89],"categorization,":[90],"discussion":[91],"limitations,":[93],"trade-offs.":[95],"Results:":[96],"Results":[97],"corroborate":[98],"critiques":[100],"management":[104],"art.":[108],"Threats":[109],"result":[110],"be":[112],"seldom":[113],"discussed":[114],"depth,":[116],"are":[118],"mostly":[119],"considered":[120],"as":[121],"an":[122,127],"enforced":[123],"afterthought":[124],"rather":[125],"than":[126],"active":[128],"concern":[129],"research":[132],"design":[133],"execution.":[135],"Conclusions:":[136],"To":[137],"improve":[138],"observed":[140],"practice,":[141],"derived":[143],"items":[144],"for":[147,154],"researchers,":[148],"reviewers":[149],"readers,":[151],"call":[153],"community":[156],"action":[157],"increase":[159],"understanding":[161],"knowledge":[163],"creation":[164]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":9}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}