{"id":"https://openalex.org/W4297152057","doi":"https://doi.org/10.3390/software1040017","title":"Empirical Formal Methods: Guidelines for Performing Empirical Studies on Formal Methods","display_name":"Empirical Formal Methods: Guidelines for Performing Empirical Studies on Formal Methods","publication_year":2022,"publication_date":"2022-09-24","ids":{"openalex":"https://openalex.org/W4297152057","doi":"https://doi.org/10.3390/software1040017"},"language":"en","primary_location":{"id":"doi:10.3390/software1040017","is_oa":true,"landing_page_url":"https://doi.org/10.3390/software1040017","pdf_url":"https://www.mdpi.com/2674-113X/1/4/17/pdf?version=1666061449","source":{"id":"https://openalex.org/S4210183365","display_name":"Software","issn_l":"2674-113X","issn":["2674-113X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://www.mdpi.com/2674-113X/1/4/17/pdf?version=1666061449","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064415269","display_name":"Maurice H. ter Beek","orcid":"https://orcid.org/0000-0002-2930-6367"},"institutions":[{"id":"https://openalex.org/I122991210","display_name":"Istituto di Scienza e Tecnologie dell'Informazione \"Alessandro Faedo\"","ror":"https://ror.org/05kacka20","country_code":"IT","type":"facility","lineage":["https://openalex.org/I122991210","https://openalex.org/I4210155236"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Maurice H. ter Beek","raw_affiliation_strings":["Formal Methods and Tools Lab, ISTI\u2013CNR, Via G. Moruzzi 1, 56124 Pisa, Italy"],"affiliations":[{"raw_affiliation_string":"Formal Methods and Tools Lab, ISTI\u2013CNR, Via G. Moruzzi 1, 56124 Pisa, Italy","institution_ids":["https://openalex.org/I122991210"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040962321","display_name":"Alessio Ferrari","orcid":"https://orcid.org/0000-0002-0636-5663"},"institutions":[{"id":"https://openalex.org/I122991210","display_name":"Istituto di Scienza e Tecnologie dell'Informazione \"Alessandro Faedo\"","ror":"https://ror.org/05kacka20","country_code":"IT","type":"facility","lineage":["https://openalex.org/I122991210","https://openalex.org/I4210155236"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Alessio Ferrari","raw_affiliation_strings":["Formal Methods and Tools Lab, ISTI\u2013CNR, Via G. Moruzzi 1, 56124 Pisa, Italy"],"affiliations":[{"raw_affiliation_string":"Formal Methods and Tools Lab, ISTI\u2013CNR, Via G. Moruzzi 1, 56124 Pisa, Italy","institution_ids":["https://openalex.org/I122991210"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5040962321"],"corresponding_institution_ids":["https://openalex.org/I122991210"],"apc_list":null,"apc_paid":null,"fwci":3.1881,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.93086329,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"1","issue":"4","first_page":"381","last_page":"416"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10142","display_name":"Formal Methods in Verification","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6737459301948547},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.6507488489151001},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.6414260864257812},{"id":"https://openalex.org/keywords/formal-methods","display_name":"Formal methods","score":0.5629078149795532},{"id":"https://openalex.org/keywords/judgement","display_name":"Judgement","score":0.47741782665252686},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.43677079677581787},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.42692068219184875},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.37892964482307434},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.359266459941864},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.14694994688034058},{"id":"https://openalex.org/keywords/medline","display_name":"MEDLINE","score":0.09939819574356079},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09900224208831787},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08726179599761963},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.07967150211334229}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6737459301948547},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.6507488489151001},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.6414260864257812},{"id":"https://openalex.org/C75606506","wikidata":"https://www.wikidata.org/wiki/Q1049183","display_name":"Formal methods","level":2,"score":0.5629078149795532},{"id":"https://openalex.org/C2776548248","wikidata":"https://www.wikidata.org/wiki/Q12621536","display_name":"Judgement","level":2,"score":0.47741782665252686},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.43677079677581787},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.42692068219184875},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.37892964482307434},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.359266459941864},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.14694994688034058},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.09939819574356079},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09900224208831787},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08726179599761963},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.07967150211334229},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3390/software1040017","is_oa":true,"landing_page_url":"https://doi.org/10.3390/software1040017","pdf_url":"https://www.mdpi.com/2674-113X/1/4/17/pdf?version=1666061449","source":{"id":"https://openalex.org/S4210183365","display_name":"Software","issn_l":"2674-113X","issn":["2674-113X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software","raw_type":"journal-article"},{"id":"pmh:oai:dnet:people______::379233ea72fe233158aa142b93e8adb7","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S7407055261","display_name":"ISTI Open Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"SOFTWARE, vol. 1 (issue 4), pp. 381-416","raw_type":"Journal article"}],"best_oa_location":{"id":"doi:10.3390/software1040017","is_oa":true,"landing_page_url":"https://doi.org/10.3390/software1040017","pdf_url":"https://www.mdpi.com/2674-113X/1/4/17/pdf?version=1666061449","source":{"id":"https://openalex.org/S4210183365","display_name":"Software","issn_l":"2674-113X","issn":["2674-113X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.46000000834465027}],"awards":[{"id":"https://openalex.org/G4456522922","display_name":null,"funder_award_id":"PRIN 2020","funder_id":"https://openalex.org/F4320321873","funder_display_name":"Ministero dell\u2019Istruzione, dell\u2019Universit\u00e0 e della Ricerca"},{"id":"https://openalex.org/G7212573297","display_name":null,"funder_award_id":"PRIN 2020TL3X8X","funder_id":"https://openalex.org/F4320321873","funder_display_name":"Ministero dell\u2019Istruzione, dell\u2019Universit\u00e0 e della Ricerca"},{"id":"https://openalex.org/G7320894718","display_name":null,"funder_award_id":"2020TL3X8X","funder_id":"https://openalex.org/F4320321873","funder_display_name":"Ministero dell\u2019Istruzione, dell\u2019Universit\u00e0 e della Ricerca"},{"id":"https://openalex.org/G895849949","display_name":null,"funder_award_id":"MIUR-PRIN","funder_id":"https://openalex.org/F4320321873","funder_display_name":"Ministero dell\u2019Istruzione, dell\u2019Universit\u00e0 e della Ricerca"}],"funders":[{"id":"https://openalex.org/F4320321873","display_name":"Ministero dell\u2019Istruzione, dell\u2019Universit\u00e0 e della Ricerca","ror":"https://ror.org/0166hxq48"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4297152057.pdf","grobid_xml":"https://content.openalex.org/works/W4297152057.grobid-xml"},"referenced_works_count":199,"referenced_works":["https://openalex.org/W49973284","https://openalex.org/W230555665","https://openalex.org/W333834217","https://openalex.org/W1521415124","https://openalex.org/W1525223041","https://openalex.org/W1564623840","https://openalex.org/W1568529421","https://openalex.org/W1582439588","https://openalex.org/W1585742531","https://openalex.org/W1603233098","https://openalex.org/W1607675442","https://openalex.org/W1608814077","https://openalex.org/W1741321747","https://openalex.org/W1791587663","https://openalex.org/W1802247608","https://openalex.org/W1918280324","https://openalex.org/W1937768634","https://openalex.org/W1967766053","https://openalex.org/W1969939902","https://openalex.org/W1975675278","https://openalex.org/W1975988951","https://openalex.org/W1978374817","https://openalex.org/W1979290264","https://openalex.org/W1983211981","https://openalex.org/W1989287546","https://openalex.org/W1990580455","https://openalex.org/W1997221584","https://openalex.org/W1999798506","https://openalex.org/W2000538740","https://openalex.org/W2001027224","https://openalex.org/W2001677911","https://openalex.org/W2006003012","https://openalex.org/W2007367842","https://openalex.org/W2007969951","https://openalex.org/W2016022053","https://openalex.org/W2020505539","https://openalex.org/W2024509488","https://openalex.org/W2030220885","https://openalex.org/W2032593675","https://openalex.org/W2034628356","https://openalex.org/W2035438947","https://openalex.org/W2042517582","https://openalex.org/W2060921624","https://openalex.org/W2064189581","https://openalex.org/W2071625344","https://openalex.org/W2075949054","https://openalex.org/W2077980038","https://openalex.org/W2084429940","https://openalex.org/W2084642972","https://openalex.org/W2087898864","https://openalex.org/W2092137831","https://openalex.org/W2092351557","https://openalex.org/W2094442885","https://openalex.org/W2098419850","https://openalex.org/W2101623580","https://openalex.org/W2104858895","https://openalex.org/W2105005227","https://openalex.org/W2106417311","https://openalex.org/W2106956101","https://openalex.org/W2111224220","https://openalex.org/W2117064875","https://openalex.org/W2118328848","https://openalex.org/W2122381799","https://openalex.org/W2122786779","https://openalex.org/W2127118596","https://openalex.org/W2133247286","https://openalex.org/W2133933998","https://openalex.org/W2136312040","https://openalex.org/W2139539295","https://openalex.org/W2144625195","https://openalex.org/W2146024157","https://openalex.org/W2146679277","https://openalex.org/W2147839365","https://openalex.org/W2150189917","https://openalex.org/W2151113201","https://openalex.org/W2151365953","https://openalex.org/W2152482417","https://openalex.org/W2153577321","https://openalex.org/W2157392897","https://openalex.org/W2160096583","https://openalex.org/W2161256170","https://openalex.org/W2162598207","https://openalex.org/W2163851162","https://openalex.org/W2168633587","https://openalex.org/W2260766877","https://openalex.org/W2281407436","https://openalex.org/W2301050748","https://openalex.org/W2344801155","https://openalex.org/W2353739181","https://openalex.org/W2362569215","https://openalex.org/W2407772516","https://openalex.org/W2477695542","https://openalex.org/W2486947741","https://openalex.org/W2518473846","https://openalex.org/W2522462291","https://openalex.org/W2576494997","https://openalex.org/W2577590554","https://openalex.org/W2590915455","https://openalex.org/W2620300258","https://openalex.org/W2735864825","https://openalex.org/W2735995639","https://openalex.org/W2755269200","https://openalex.org/W2765527199","https://openalex.org/W2771581566","https://openalex.org/W2787918416","https://openalex.org/W2788267443","https://openalex.org/W2788276534","https://openalex.org/W2794237666","https://openalex.org/W2801076109","https://openalex.org/W2810607235","https://openalex.org/W2884325678","https://openalex.org/W2885628182","https://openalex.org/W2889849786","https://openalex.org/W2890801208","https://openalex.org/W2890861406","https://openalex.org/W2891421390","https://openalex.org/W2897889469","https://openalex.org/W2900780837","https://openalex.org/W2904533741","https://openalex.org/W2905180391","https://openalex.org/W2910647970","https://openalex.org/W2914024328","https://openalex.org/W2917615073","https://openalex.org/W2932603064","https://openalex.org/W2946384079","https://openalex.org/W2949189536","https://openalex.org/W2953557274","https://openalex.org/W2954754777","https://openalex.org/W2955510165","https://openalex.org/W2957557853","https://openalex.org/W2967916802","https://openalex.org/W2968013754","https://openalex.org/W2969699495","https://openalex.org/W2974906992","https://openalex.org/W2980548593","https://openalex.org/W2986478207","https://openalex.org/W2990676673","https://openalex.org/W2991443836","https://openalex.org/W3006568433","https://openalex.org/W3010114092","https://openalex.org/W3014560488","https://openalex.org/W3021098898","https://openalex.org/W3023605951","https://openalex.org/W3025796623","https://openalex.org/W3034832379","https://openalex.org/W3036803033","https://openalex.org/W3037603558","https://openalex.org/W3037615466","https://openalex.org/W3049513310","https://openalex.org/W3082634688","https://openalex.org/W3084301791","https://openalex.org/W3087764159","https://openalex.org/W3088417404","https://openalex.org/W3088608579","https://openalex.org/W3088927226","https://openalex.org/W3105185804","https://openalex.org/W3106266288","https://openalex.org/W3115079017","https://openalex.org/W3123466959","https://openalex.org/W3134402730","https://openalex.org/W3134734660","https://openalex.org/W3134966439","https://openalex.org/W3135995219","https://openalex.org/W3138686503","https://openalex.org/W3140839120","https://openalex.org/W3159933051","https://openalex.org/W3162732694","https://openalex.org/W3174994947","https://openalex.org/W3195818176","https://openalex.org/W3196007188","https://openalex.org/W3202205940","https://openalex.org/W3208645270","https://openalex.org/W3211510132","https://openalex.org/W4206254927","https://openalex.org/W4206405501","https://openalex.org/W4210246416","https://openalex.org/W4214928512","https://openalex.org/W4220891756","https://openalex.org/W4225502303","https://openalex.org/W4229584947","https://openalex.org/W4233152543","https://openalex.org/W4237578926","https://openalex.org/W4237638794","https://openalex.org/W4242485331","https://openalex.org/W4288079797","https://openalex.org/W4291213652","https://openalex.org/W4296471858","https://openalex.org/W4385983939","https://openalex.org/W6609183171","https://openalex.org/W6635318388","https://openalex.org/W6636190069","https://openalex.org/W6636342884","https://openalex.org/W6680919182","https://openalex.org/W6692485200","https://openalex.org/W6756302703","https://openalex.org/W6763683228","https://openalex.org/W6767461485","https://openalex.org/W6770709529","https://openalex.org/W6812742317"],"related_works":["https://openalex.org/W2886802431","https://openalex.org/W4388216822","https://openalex.org/W1761762290","https://openalex.org/W4391191813","https://openalex.org/W4387575966","https://openalex.org/W2384262901","https://openalex.org/W2975214487","https://openalex.org/W651304006","https://openalex.org/W2345720417","https://openalex.org/W2385564112"],"abstract_inverted_index":{"Empirical":[0],"studies":[1,88],"on":[2],"formal":[3,68,79,101],"methods":[4,69],"and":[5,23,36,49,53,70,85],"tools":[6],"are":[7],"rare.":[8],"In":[9],"this":[10],"paper,":[11],"we":[12],"provide":[13],"guidelines":[14],"for":[15,55,99],"such":[16],"studies.":[17],"We":[18,92],"mention":[19],"their":[20,59,76],"main":[21],"ingredients":[22],"then":[24],"define":[25],"nine":[26],"different":[27],"study":[28],"strategies":[29],"(usability":[30],"testing,":[31],"laboratory":[32],"experiments":[33],"with":[34,94],"software":[35],"human":[37],"subjects,":[38],"case":[39],"studies,":[40,42,45],"qualitative":[41],"surveys,":[43],"judgement":[44],"systematic":[46,50],"literature":[47],"reviews,":[48],"mapping":[51],"studies)":[52],"discuss":[54],"each":[56],"of":[57,64,97],"them":[58,66],"crucial":[60],"characteristics,":[61],"the":[62],"difficulties":[63],"applying":[65],"to":[67,74,82,87],"tools,":[71],"typical":[72],"threats":[73],"validity,":[75],"maturity":[77],"in":[78,89],"methods,":[80],"pointers":[81,86],"external":[83],"guidelines,":[84],"other":[90],"fields.":[91],"conclude":[93],"a":[95],"number":[96],"challenges":[98],"empirical":[100],"methods.":[102]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}