{"id":"https://openalex.org/W2159715729","doi":"https://doi.org/10.1145/1852786.1852789","title":"Can we evaluate the quality of software engineering experiments?","display_name":"Can we evaluate the quality of software engineering experiments?","publication_year":2010,"publication_date":"2010-09-14","ids":{"openalex":"https://openalex.org/W2159715729","doi":"https://doi.org/10.1145/1852786.1852789","mag":"2159715729"},"language":"en","primary_location":{"id":"doi:10.1145/1852786.1852789","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1852786.1852789","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2010 ACM-IEEE International Symposium on Empirical Software Engineering and Measurement","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://lup.lub.lu.se/record/1680334","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5012102325","display_name":"Barbara Kitchenham","orcid":"https://orcid.org/0000-0002-6134-8460"},"institutions":[{"id":"https://openalex.org/I56007636","display_name":"Keele University","ror":"https://ror.org/00340yn33","country_code":"GB","type":"education","lineage":["https://openalex.org/I56007636"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Barbara Kitchenham","raw_affiliation_strings":["Keele University, Keele, Staffordshire, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Keele University, Keele, Staffordshire, UK","institution_ids":["https://openalex.org/I56007636"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078367528","display_name":"Dag I. K. Sj\u00f8berg","orcid":"https://orcid.org/0000-0002-4941-7240"},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Dag I. K. Sj\u00f8berg","raw_affiliation_strings":["University of Oslo, Oslo, Norway"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oslo, Oslo, Norway","institution_ids":["https://openalex.org/I184942183"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012868299","display_name":"O. Pearl Brereton","orcid":null},"institutions":[{"id":"https://openalex.org/I56007636","display_name":"Keele University","ror":"https://ror.org/00340yn33","country_code":"GB","type":"education","lineage":["https://openalex.org/I56007636"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"O. Pearl Brereton","raw_affiliation_strings":["Keele University, Keele, Staffordshire, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Keele University, Keele, Staffordshire, UK","institution_ids":["https://openalex.org/I56007636"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061755008","display_name":"David Budgen","orcid":"https://orcid.org/0000-0001-7143-0241"},"institutions":[{"id":"https://openalex.org/I190082696","display_name":"Durham University","ror":"https://ror.org/01v29qb04","country_code":"GB","type":"education","lineage":["https://openalex.org/I190082696"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"David Budgen","raw_affiliation_strings":["Durham University, Durham, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Durham University, Durham, UK","institution_ids":["https://openalex.org/I190082696"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109079407","display_name":"Tore Dyb\u00e5","orcid":null},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Tore Dyb\u00e5","raw_affiliation_strings":["University of Oslo, Norway"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oslo, Norway","institution_ids":["https://openalex.org/I184942183"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045592913","display_name":"Martin H\u00f6st","orcid":"https://orcid.org/0000-0002-9360-8693"},"institutions":[{"id":"https://openalex.org/I187531555","display_name":"Lund University","ror":"https://ror.org/012a77v79","country_code":"SE","type":"education","lineage":["https://openalex.org/I187531555"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Martin H\u00f6st","raw_affiliation_strings":["Lund University, Lund, Sweden"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Lund University, Lund, Sweden","institution_ids":["https://openalex.org/I187531555"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022214666","display_name":"Dietmar Pfahl","orcid":"https://orcid.org/0000-0003-2400-501X"},"institutions":[{"id":"https://openalex.org/I168635309","display_name":"University of Calgary","ror":"https://ror.org/03yjb2x39","country_code":"CA","type":"education","lineage":["https://openalex.org/I168635309"]},{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]}],"countries":["CA","NO"],"is_corresponding":false,"raw_author_name":"Dietmar Pfahl","raw_affiliation_strings":["University of Oslo, Norway and University of Calgary, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oslo, Norway and University of Calgary, Canada","institution_ids":["https://openalex.org/I168635309","https://openalex.org/I184942183"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087032437","display_name":"Per Runeson","orcid":"https://orcid.org/0000-0003-2795-4851"},"institutions":[{"id":"https://openalex.org/I187531555","display_name":"Lund University","ror":"https://ror.org/012a77v79","country_code":"SE","type":"education","lineage":["https://openalex.org/I187531555"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Per Runeson","raw_affiliation_strings":["Lund University, Lund, Sweden"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Lund University, Lund, Sweden","institution_ids":["https://openalex.org/I187531555"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":21.4088,"has_fulltext":false,"cited_by_count":63,"citation_normalized_percentile":{"value":0.99221806,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9743000268936157,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.7232338786125183},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7136403918266296},{"id":"https://openalex.org/keywords/checklist","display_name":"Checklist","score":0.7125348448753357},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6996713876724243},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.6104539036750793},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6088035106658936},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5665110945701599},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5010483264923096},{"id":"https://openalex.org/keywords/quality-score","display_name":"Quality Score","score":0.47538888454437256},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.42602449655532837},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.40450263023376465},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.34566137194633484},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.17158448696136475},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.13954445719718933},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.13864898681640625},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.11820882558822632},{"id":"https://openalex.org/keywords/operations-management","display_name":"Operations management","score":0.10473743081092834}],"concepts":[{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.7232338786125183},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7136403918266296},{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.7125348448753357},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6996713876724243},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.6104539036750793},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6088035106658936},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5665110945701599},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5010483264923096},{"id":"https://openalex.org/C2779346075","wikidata":"https://www.wikidata.org/wiki/Q7268763","display_name":"Quality Score","level":3,"score":0.47538888454437256},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.42602449655532837},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.40450263023376465},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.34566137194633484},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.17158448696136475},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.13954445719718933},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.13864898681640625},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11820882558822632},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.10473743081092834},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/1852786.1852789","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1852786.1852789","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2010 ACM-IEEE International Symposium on Empirical Software Engineering and Measurement","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.706.9752","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.706.9752","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.cs.montana.edu/courses/esof522/handouts_papers/kitchenham-2010.pdf","raw_type":"text"},{"id":"pmh:oai:lup.lub.lu.se:53991f99-a26e-423f-bbb5-4a903f0e5a26","is_oa":true,"landing_page_url":"https://lup.lub.lu.se/record/1680334","pdf_url":null,"source":{"id":"https://openalex.org/S4306400536","display_name":"Lund University Publications (Lund University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I187531555","host_organization_name":"Lund University","host_organization_lineage":["https://openalex.org/I187531555"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:lup.lub.lu.se:53991f99-a26e-423f-bbb5-4a903f0e5a26","is_oa":true,"landing_page_url":"https://lup.lub.lu.se/record/1680334","pdf_url":null,"source":{"id":"https://openalex.org/S4306400536","display_name":"Lund University Publications (Lund University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I187531555","host_organization_name":"Lund University","host_organization_lineage":["https://openalex.org/I187531555"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W1517951603","https://openalex.org/W1557448944","https://openalex.org/W1976546926","https://openalex.org/W1977862340","https://openalex.org/W1979442556","https://openalex.org/W1980710855","https://openalex.org/W2013359927","https://openalex.org/W2038663983","https://openalex.org/W2051213584","https://openalex.org/W2063743914","https://openalex.org/W2086314066","https://openalex.org/W2110707035","https://openalex.org/W2114269784","https://openalex.org/W2123707693","https://openalex.org/W2143889788","https://openalex.org/W2146773231","https://openalex.org/W2147826933","https://openalex.org/W2154755835","https://openalex.org/W2162411541","https://openalex.org/W2168811232","https://openalex.org/W2170965652","https://openalex.org/W2176558818","https://openalex.org/W2400915478","https://openalex.org/W2463581525","https://openalex.org/W2799235570","https://openalex.org/W4246553962"],"related_works":["https://openalex.org/W4234875088","https://openalex.org/W2013796470","https://openalex.org/W2420514274","https://openalex.org/W2001467811","https://openalex.org/W4362687946","https://openalex.org/W1971633942","https://openalex.org/W2902805979","https://openalex.org/W2987114614","https://openalex.org/W2808458916","https://openalex.org/W2145923542"],"abstract_inverted_index":{"Context:":[0],"The":[1,30,106,137,190],"authors":[2],"wanted":[3],"to":[4,36],"assess":[5],"whether":[6],"the":[7,24,33,38,87,92,100,140,180,184,187,197,202,209,212,216,228,233],"quality":[8,25,42,80,156,176],"of":[9,22,26,32,40,139,169,186,193,201,211,215],"published":[10],"human-centric":[11,73],"software":[12,74],"engineering":[13,75],"experiments":[14],"was":[15,94,109,143],"improving.":[16],"This":[17],"required":[18,165],"a":[19,41,79,150,167,224],"reliable":[20,130,145],"means":[21],"assessing":[23],"such":[27],"experiments.":[28],"Aims:":[29],"aims":[31],"study":[34,88,217],"were":[35,49,128],"confirm":[37],"usability":[39],"evaluation":[43,157],"checklist,":[44],"determine":[45],"how":[46],"many":[47],"reviewers":[48,68,121,133,163,203],"needed":[50],"per":[51],"paper":[52],"that":[53],"reports":[54],"an":[55,59],"experiment,":[56],"and":[57,69,99,166,178,196],"specify":[58],"appropriate":[60],"process":[61],"for":[62,111,117],"evaluating":[63],"quality.":[64],"Method:":[65],"With":[66],"eight":[67,132],"four":[70],"papers":[71,194],"describing":[72],"experiments,":[76],"we":[77],"used":[78,195],"checklist":[81],"with":[82,126,134],"nine":[83,141],"questions.":[84],"We":[85,173],"conducted":[86],"in":[89,123],"two":[90,124,162],"parts:":[91],"first":[93],"based":[95],"on":[96,102,183,227],"individual":[97,112,147],"assessments":[98,113],"second":[101,213],"collaborative":[103],"evaluations.":[104,119],"Results:":[105],"inter-rater":[107],"reliability":[108],"poor":[110],"but":[114],"much":[115],"better":[116],"joint":[118],"Four":[120],"working":[122],"pairs":[125],"discussion":[127,170],"more":[129,144,160],"than":[131,146,161],"no":[135],"discussion.":[136],"sum":[138,185],"criteria":[142,177],"questions":[148],"or":[149],"simple":[151],"overall":[152],"assessment.":[153],"Conclusions:":[154],"If":[155],"is":[158,171],"critical,":[159],"are":[164],"round":[168],"necessary.":[172],"advise":[174],"using":[175],"basing":[179],"final":[181],"assessment":[182],"aggregated":[188],"criteria.":[189],"restricted":[191],"number":[192],"relatively":[198],"extensive":[199],"expertise":[200],"limit":[204],"our":[205],"results.":[206],"In":[207],"addition,":[208],"results":[210],"part":[214],"could":[218],"have":[219],"been":[220],"affected":[221],"by":[222],"removing":[223],"time":[225],"restriction":[226],"review":[229],"as":[230,232],"well":[231],"consultation":[234],"process.":[235]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":7},{"year":2016,"cited_by_count":3},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":3},{"year":2013,"cited_by_count":4},{"year":2012,"cited_by_count":17}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}