{"id":"https://openalex.org/W2940439009","doi":"https://doi.org/10.1145/3319008.3319018","title":"Practitioner Evaluations on Software Testing Tools","display_name":"Practitioner Evaluations on Software Testing Tools","publication_year":2019,"publication_date":"2019-04-10","ids":{"openalex":"https://openalex.org/W2940439009","doi":"https://doi.org/10.1145/3319008.3319018","mag":"2940439009"},"language":"en","primary_location":{"id":"doi:10.1145/3319008.3319018","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3319008.3319018","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Evaluation and Assessment on Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014755653","display_name":"P\u00e4ivi Raulamo-Jurvanen","orcid":null},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":true,"raw_author_name":"P\u00e4ivi Raulamo-Jurvanen","raw_affiliation_strings":["M3S, University of Oulu, Oulu, Finland"],"affiliations":[{"raw_affiliation_string":"M3S, University of Oulu, Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067689431","display_name":"Simo Hosio","orcid":"https://orcid.org/0000-0002-9609-0965"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Simo Hosio","raw_affiliation_strings":["UBICOMP, University of Oulu, Oulu, Finland"],"affiliations":[{"raw_affiliation_string":"UBICOMP, University of Oulu, Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078824435","display_name":"Mika M\u00e4ntyl\u00e4","orcid":"https://orcid.org/0000-0002-2841-5879"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Mika V. M\u00e4ntyl\u00e4","raw_affiliation_strings":["M3S, University of Oulu, Oulu, Finland"],"affiliations":[{"raw_affiliation_string":"M3S, University of Oulu, Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5014755653"],"corresponding_institution_ids":["https://openalex.org/I98381234"],"apc_list":null,"apc_paid":null,"fwci":3.0523,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.92628663,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"57","last_page":"66"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/regression-testing","display_name":"Regression testing","score":0.5998647212982178},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5646771192550659},{"id":"https://openalex.org/keywords/demographics","display_name":"Demographics","score":0.5540659427642822},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5358214378356934},{"id":"https://openalex.org/keywords/seniority","display_name":"Seniority","score":0.5163856148719788},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.5128152966499329},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.44775834679603577},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.44204485416412354},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3924407362937927},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.33677399158477783},{"id":"https://openalex.org/keywords/software-construction","display_name":"Software construction","score":0.16811048984527588},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.16409218311309814},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.16160506010055542}],"concepts":[{"id":"https://openalex.org/C161821725","wikidata":"https://www.wikidata.org/wiki/Q917415","display_name":"Regression testing","level":5,"score":0.5998647212982178},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5646771192550659},{"id":"https://openalex.org/C2780084366","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demographics","level":2,"score":0.5540659427642822},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5358214378356934},{"id":"https://openalex.org/C2780783599","wikidata":"https://www.wikidata.org/wiki/Q1931779","display_name":"Seniority","level":2,"score":0.5163856148719788},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5128152966499329},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.44775834679603577},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.44204485416412354},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3924407362937927},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.33677399158477783},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.16811048984527588},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.16409218311309814},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.16160506010055542},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3319008.3319018","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3319008.3319018","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Evaluation and Assessment on Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:oulu.fi:nbnfi-fe202001142020","is_oa":false,"landing_page_url":"http://urn.fi/urn:nbn:fi-fe202001142020","pdf_url":null,"source":{"id":"https://openalex.org/S4306400284","display_name":"University of Oulu Repository (University of Oulu)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98381234","host_organization_name":"University of Oulu","host_organization_lineage":["https://openalex.org/I98381234"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320325600","display_name":"ITEA3","ror":null},{"id":"https://openalex.org/F4320338388","display_name":"Eurostars","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":48,"referenced_works":["https://openalex.org/W115394847","https://openalex.org/W136732505","https://openalex.org/W171406816","https://openalex.org/W171938871","https://openalex.org/W1600487731","https://openalex.org/W1692158061","https://openalex.org/W1754212385","https://openalex.org/W1971139216","https://openalex.org/W1971776182","https://openalex.org/W1987076101","https://openalex.org/W2000869978","https://openalex.org/W2011399107","https://openalex.org/W2011808739","https://openalex.org/W2016499703","https://openalex.org/W2035099508","https://openalex.org/W2072404325","https://openalex.org/W2077938752","https://openalex.org/W2092198121","https://openalex.org/W2097582896","https://openalex.org/W2114647523","https://openalex.org/W2122646361","https://openalex.org/W2141403362","https://openalex.org/W2149142774","https://openalex.org/W2152361512","https://openalex.org/W2153222072","https://openalex.org/W2159839830","https://openalex.org/W2163851162","https://openalex.org/W2169250200","https://openalex.org/W2172104240","https://openalex.org/W2319794630","https://openalex.org/W2327037637","https://openalex.org/W2345039647","https://openalex.org/W2380271241","https://openalex.org/W2472091773","https://openalex.org/W2513796004","https://openalex.org/W2519446413","https://openalex.org/W2546756781","https://openalex.org/W2554291695","https://openalex.org/W2561938281","https://openalex.org/W2579878149","https://openalex.org/W2590256340","https://openalex.org/W2613884210","https://openalex.org/W2620872098","https://openalex.org/W2750190833","https://openalex.org/W2797630187","https://openalex.org/W2989567029","https://openalex.org/W3005800746","https://openalex.org/W4210764005"],"related_works":["https://openalex.org/W3199041638","https://openalex.org/W2589723489","https://openalex.org/W1923185232","https://openalex.org/W2625118345","https://openalex.org/W3209487211","https://openalex.org/W2184708610","https://openalex.org/W4384009674","https://openalex.org/W2617144600","https://openalex.org/W2106191864","https://openalex.org/W2062249491"],"abstract_inverted_index":{"In":[0,24],"software":[1,8,28,56],"engineering":[2],"practice,":[3],"evaluating":[4],"and":[5,21,32,38,81,158,173],"selecting":[6],"the":[7,14,69,72,93,112,117,120,155,161],"testing":[9,59],"tools":[10],"that":[11,42,99,128,145],"best":[12],"fit":[13],"project":[15],"at":[16],"hand":[17],"is":[18],"an":[19],"important":[20],"challenging":[22],"task.":[23],"scientific":[25],"studies":[26,40,168],"of":[27,46,71,83,95,114,119,139,160],"engineering,":[29],"practitioner":[30],"evaluations":[31],"beliefs":[33,45],"have":[34],"recently":[35],"gained":[36],"interest,":[37],"some":[39],"suggest":[41],"practitioners":[43,57],"find":[44,98,144],"peers":[47],"more":[48,150],"credible":[49],"than":[50],"empirical":[51],"evidence.":[52],"To":[53,153],"study":[54],"how":[55,110],"evaluate":[58,92],"tools,":[60],"we":[61,143,163],"applied":[62],"online":[63],"opinion":[64],"surveys":[65],"(n=89).":[66],"We":[67,97,108],"analyzed":[68],"reliability":[70,118],"opinions":[73,100,131],"utilizing":[74,170],"Krippendorff's":[75],"alpha,":[76],"intra-class":[77],"correlation":[78],"coefficient":[79],"(ICC),":[80],"coefficients":[82],"variation":[84],"(CV).":[85],"Negative":[86],"binomial":[87],"regression":[88],"was":[89],"used":[90],"to":[91,149,165],"effect":[94],"demographics.":[96],"towards":[101],"a":[102,136],"specific":[103],"tool":[104],"can":[105],"be":[106],"conflicting.":[107],"show":[109],"increasing":[111],"number":[113],"respondents":[115],"improves":[116],"estimates":[121],"measured":[122],"with":[123],"ICC.":[124],"Our":[125],"results":[126],"indicate":[127],"on":[129],"average,":[130],"from":[132],"seven":[133],"experts":[134],"provide":[135],"moderate":[137],"level":[138],"reliability.":[140],"From":[141],"demographics,":[142],"technical":[146],"seniority":[147],"leads":[148],"negative":[151],"evaluations.":[152],"improve":[154],"understanding,":[156],"robustness,":[157],"impact":[159],"findings,":[162],"need":[164],"conduct":[166],"further":[167],"by":[169],"diverse":[171],"sources":[172],"complementary":[174],"methods.":[175]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}