{"id":"https://openalex.org/W2237709800","doi":"https://doi.org/10.1080/07421222.2015.1099390","title":"A Taxonomy of Evaluation Methods for Information Systems Artifacts","display_name":"A Taxonomy of Evaluation Methods for Information Systems Artifacts","publication_year":2015,"publication_date":"2015-07-03","ids":{"openalex":"https://openalex.org/W2237709800","doi":"https://doi.org/10.1080/07421222.2015.1099390","mag":"2237709800"},"language":"en","primary_location":{"id":"doi:10.1080/07421222.2015.1099390","is_oa":false,"landing_page_url":"https://doi.org/10.1080/07421222.2015.1099390","pdf_url":null,"source":{"id":"https://openalex.org/S9954729","display_name":"Journal of Management Information Systems","issn_l":"0742-1222","issn":["0742-1222","1557-928X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547"],"host_organization_lineage_names":["Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Management Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102716936","display_name":"Nicolas Prat","orcid":"https://orcid.org/0000-0002-8064-9357"},"institutions":[{"id":"https://openalex.org/I4210142324","display_name":"CY Cergy Paris Universit\u00e9","ror":"https://ror.org/043htjv09","country_code":"FR","type":"education","lineage":["https://openalex.org/I4210142324"]},{"id":"https://openalex.org/I124261006","display_name":"\u00c9cole Sup\u00e9rieure des Sciences \u00c9conomiques et Commerciales","ror":"https://ror.org/02dga6j42","country_code":"FR","type":"other","lineage":["https://openalex.org/I124261006"]}],"countries":["FR"],"is_corresponding":true,"raw_author_name":"Nicolas Prat","raw_affiliation_strings":["View further author information","ESSEC Business School (1, avenue Bernard Hirsch\r\nCS 50105 Cergy\r\n95021 Cergy Pontoise Cedex\r\nFrance\r\nT\u00e9l : +33 (0)1 34 43 30 00\r\nFax : +33 (0)1 34 43 30 01 - France)"],"affiliations":[{"raw_affiliation_string":"View further author information","institution_ids":[]},{"raw_affiliation_string":"ESSEC Business School (1, avenue Bernard Hirsch\r\nCS 50105 Cergy\r\n95021 Cergy Pontoise Cedex\r\nFrance\r\nT\u00e9l : +33 (0)1 34 43 30 00\r\nFax : +33 (0)1 34 43 30 01 - France)","institution_ids":["https://openalex.org/I4210142324","https://openalex.org/I124261006"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029527940","display_name":"Isabelle Comyn-Wattiau","orcid":"https://orcid.org/0000-0002-5257-6189"},"institutions":[{"id":"https://openalex.org/I4210145724","display_name":"Centre d'Etudes et De Recherche en Informatique et Communications","ror":"https://ror.org/044j5mm75","country_code":"FR","type":"facility","lineage":["https://openalex.org/I4210145724"]},{"id":"https://openalex.org/I4210142324","display_name":"CY Cergy Paris Universit\u00e9","ror":"https://ror.org/043htjv09","country_code":"FR","type":"education","lineage":["https://openalex.org/I4210142324"]},{"id":"https://openalex.org/I124261006","display_name":"\u00c9cole Sup\u00e9rieure des Sciences \u00c9conomiques et Commerciales","ror":"https://ror.org/02dga6j42","country_code":"FR","type":"other","lineage":["https://openalex.org/I124261006"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Isabelle Comyn-Wattiau","raw_affiliation_strings":["View further author information","CEDRIC - Centre d'\u00e9tudes et de recherche en informatique et communications (292 rue Saint-Martin \r\n75141 PARIS Cedex 03\r\nCase courrier 2D4P30 - France)","ESSEC Business School (1, avenue Bernard Hirsch\r\nCS 50105 Cergy\r\n95021 Cergy Pontoise Cedex\r\nFrance\r\nT\u00e9l : +33 (0)1 34 43 30 00\r\nFax : +33 (0)1 34 43 30 01 - France)"],"affiliations":[{"raw_affiliation_string":"View further author information","institution_ids":[]},{"raw_affiliation_string":"CEDRIC - Centre d'\u00e9tudes et de recherche en informatique et communications (292 rue Saint-Martin \r\n75141 PARIS Cedex 03\r\nCase courrier 2D4P30 - France)","institution_ids":["https://openalex.org/I4210145724"]},{"raw_affiliation_string":"ESSEC Business School (1, avenue Bernard Hirsch\r\nCS 50105 Cergy\r\n95021 Cergy Pontoise Cedex\r\nFrance\r\nT\u00e9l : +33 (0)1 34 43 30 00\r\nFax : +33 (0)1 34 43 30 01 - France)","institution_ids":["https://openalex.org/I4210142324","https://openalex.org/I124261006"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001917118","display_name":"Jacky Akoka","orcid":"https://orcid.org/0000-0002-3228-3266"},"institutions":[{"id":"https://openalex.org/I4210145724","display_name":"Centre d'Etudes et De Recherche en Informatique et Communications","ror":"https://ror.org/044j5mm75","country_code":"FR","type":"facility","lineage":["https://openalex.org/I4210145724"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Jacky Akoka","raw_affiliation_strings":["View further author information","CEDRIC - ISID - CEDRIC. Ing\u00e9nierie des Syst\u00e8mes d'Information et de D\u00e9cision (France)","IMT-BS - DSI - D\u00e9partement Syst\u00e8mes d'Information (9, rue Charles Fourier - 91011 Evry Cedex - France)"],"affiliations":[{"raw_affiliation_string":"View further author information","institution_ids":[]},{"raw_affiliation_string":"CEDRIC - ISID - CEDRIC. Ing\u00e9nierie des Syst\u00e8mes d'Information et de D\u00e9cision (France)","institution_ids":["https://openalex.org/I4210145724"]},{"raw_affiliation_string":"IMT-BS - DSI - D\u00e9partement Syst\u00e8mes d'Information (9, rue Charles Fourier - 91011 Evry Cedex - France)","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5102716936"],"corresponding_institution_ids":["https://openalex.org/I124261006","https://openalex.org/I4210142324"],"apc_list":null,"apc_paid":null,"fwci":40.8294,"has_fulltext":false,"cited_by_count":245,"citation_normalized_percentile":{"value":0.99729065,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"32","issue":"3","first_page":"229","last_page":"267"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11024","display_name":"Information Systems Theories and Implementation","score":0.9952999949455261,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11024","display_name":"Information Systems Theories and Implementation","score":0.9952999949455261,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11891","display_name":"Big Data and Business Intelligence","score":0.984499990940094,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/artifact","display_name":"Artifact (error)","score":0.7878508567810059},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.761168897151947},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.5868918895721436},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.5667272806167603},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.5114566087722778},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5053008198738098},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.46216559410095215},{"id":"https://openalex.org/keywords/design-science-research","display_name":"Design science research","score":0.43990570306777954},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.4053305983543396},{"id":"https://openalex.org/keywords/information-system","display_name":"Information system","score":0.3400232195854187},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.32683414220809937},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.29499363899230957},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10254964232444763},{"id":"https://openalex.org/keywords/management","display_name":"Management","score":0.0828251838684082}],"concepts":[{"id":"https://openalex.org/C2779010991","wikidata":"https://www.wikidata.org/wiki/Q2720909","display_name":"Artifact (error)","level":2,"score":0.7878508567810059},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.761168897151947},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.5868918895721436},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.5667272806167603},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5114566087722778},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5053008198738098},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.46216559410095215},{"id":"https://openalex.org/C2777655877","wikidata":"https://www.wikidata.org/wiki/Q3475740","display_name":"Design science research","level":3,"score":0.43990570306777954},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.4053305983543396},{"id":"https://openalex.org/C180198813","wikidata":"https://www.wikidata.org/wiki/Q121182","display_name":"Information system","level":2,"score":0.3400232195854187},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.32683414220809937},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29499363899230957},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10254964232444763},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0828251838684082},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1080/07421222.2015.1099390","is_oa":false,"landing_page_url":"https://doi.org/10.1080/07421222.2015.1099390","pdf_url":null,"source":{"id":"https://openalex.org/S9954729","display_name":"Journal of Management Information Systems","issn_l":"0742-1222","issn":["0742-1222","1557-928X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547"],"host_organization_lineage_names":["Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Management Information Systems","raw_type":"journal-article"},{"id":"pmh:oai:HAL:hal-01272048v1","is_oa":false,"landing_page_url":"https://hal.science/hal-01272048","pdf_url":null,"source":{"id":"https://openalex.org/S4406922466","display_name":"SPIRE - Sciences Po Institutional REpository","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Journal of Management Information Systems, 2015, 32 (3), pp.229 - 267. ⟨10.1080/07421222.2015.1099390⟩","raw_type":"Journal articles"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":77,"referenced_works":["https://openalex.org/W18209118","https://openalex.org/W50221525","https://openalex.org/W62824994","https://openalex.org/W124349154","https://openalex.org/W127911944","https://openalex.org/W156509383","https://openalex.org/W159389675","https://openalex.org/W178774342","https://openalex.org/W261250222","https://openalex.org/W573546480","https://openalex.org/W602489364","https://openalex.org/W637844663","https://openalex.org/W1276866904","https://openalex.org/W1449700966","https://openalex.org/W1507037530","https://openalex.org/W1520980263","https://openalex.org/W1541456620","https://openalex.org/W1560620786","https://openalex.org/W1567491469","https://openalex.org/W1620479424","https://openalex.org/W1748363735","https://openalex.org/W1791587663","https://openalex.org/W1963620879","https://openalex.org/W1980446154","https://openalex.org/W1980902933","https://openalex.org/W1983749049","https://openalex.org/W1993285562","https://openalex.org/W1995156820","https://openalex.org/W1996794788","https://openalex.org/W2007776423","https://openalex.org/W2023268087","https://openalex.org/W2026948646","https://openalex.org/W2037808021","https://openalex.org/W2042300236","https://openalex.org/W2053680135","https://openalex.org/W2057012437","https://openalex.org/W2062114445","https://openalex.org/W2075959942","https://openalex.org/W2076622132","https://openalex.org/W2079484041","https://openalex.org/W2088961263","https://openalex.org/W2100408980","https://openalex.org/W2104112667","https://openalex.org/W2107604945","https://openalex.org/W2107928912","https://openalex.org/W2114984213","https://openalex.org/W2116966683","https://openalex.org/W2123601949","https://openalex.org/W2133610185","https://openalex.org/W2134436159","https://openalex.org/W2134934846","https://openalex.org/W2136451344","https://openalex.org/W2136922540","https://openalex.org/W2138073318","https://openalex.org/W2143015893","https://openalex.org/W2144683585","https://openalex.org/W2146449629","https://openalex.org/W2148398334","https://openalex.org/W2151020819","https://openalex.org/W2151461605","https://openalex.org/W2158342889","https://openalex.org/W2169571673","https://openalex.org/W2186301631","https://openalex.org/W2188809343","https://openalex.org/W2203717586","https://openalex.org/W2402567235","https://openalex.org/W2528763179","https://openalex.org/W2582743722","https://openalex.org/W2945396417","https://openalex.org/W3023448671","https://openalex.org/W3123414772","https://openalex.org/W3151685851","https://openalex.org/W4239329905","https://openalex.org/W4248450049","https://openalex.org/W4252952299","https://openalex.org/W4254412034","https://openalex.org/W4300114914"],"related_works":["https://openalex.org/W4385231812","https://openalex.org/W1741504538","https://openalex.org/W3021294361","https://openalex.org/W2497647047","https://openalex.org/W4310103254","https://openalex.org/W4285234616","https://openalex.org/W2468542265","https://openalex.org/W3006572021","https://openalex.org/W1139307540","https://openalex.org/W2010723890"],"abstract_inverted_index":{"Artifacts,":[0],"such":[1],"as":[2,128],"software":[3],"systems,":[4],"pervade":[5],"organizations":[6],"and":[7,40,73,84,94,100,162,170,180,199,233],"society.":[8],"In":[9],"the":[10,18,46,71,74,82,88,92,95,98,101,137,142,156,229,234],"field":[11],"of":[12,20,24,48,65,76,86,103,113,133,139,141,158,173,177,184,188,192,197,209,227,237],"information":[13],"systems":[14],"(IS)":[15],"they":[16],"form":[17],"core":[19],"research.":[21],"The":[22],"evaluation":[23,114,126,166,176,183,187,196,211],"IS":[25,33,55,77,117,124,159,220],"artifacts":[26,42],"thus":[27],"represents":[28],"a":[29,111,207,214],"major":[30],"issue.":[31],"Although":[32],"research":[34,51,149],"paradigms":[35],"are":[36,81,222],"increasingly":[37],"intertwined,":[38],"building":[39],"evaluating":[41,91,228],"has":[43,56],"traditionally":[44],"been":[45],"purview":[47],"design":[49],"science":[50],"(DSR).":[52],"DSR":[53,134],"in":[54,136],"not":[57],"reached":[58],"maturity":[59],"yet.":[60],"This":[61,68,148,203],"is":[62],"particularly":[63],"true":[64],"artifact":[66,78,125,160,210],"evaluation.":[67],"paper":[69],"investigates":[70],"\u201cwhat\u201d":[72,99],"\u201chow\u201d":[75,102],"evaluation:":[79],"what":[80],"objects":[83],"criteria":[85],"evaluation,":[87,161],"methods":[89,115],"for":[90,116,144],"criteria,":[93],"relationships":[96,154],"between":[97,155],"evaluation?":[104],"To":[105],"answer":[106],"these":[107],"questions,":[108],"we":[109,122],"develop":[110],"taxonomy":[112],"artifacts.":[118,238],"With":[119],"this":[120],"taxonomy,":[121],"analyze":[123],"practice,":[127],"reflected":[129],"by":[130],"ten":[131],"years":[132],"publications":[135],"basket":[138],"journals":[140],"Association":[143],"Information":[145],"Systems":[146],"(AIS).":[147],"brings":[150],"to":[151,224],"light":[152],"important":[153],"dimensions":[157],"identifies":[163],"seven":[164],"typical":[165],"patterns:":[167],"demonstration;":[168],"simulation-":[169,179],"metric-based":[171,181],"benchmarking":[172],"artifacts;":[174,185],"practice-based":[175,186],"effectiveness;":[178],"absolute":[182],"usefulness":[189],"or":[190],"ease":[191],"use;":[193],"laboratory,":[194],"student-based":[195],"usefulness;":[198],"algorithmic":[200],"complexity":[201],"analysis.":[202],"study":[204],"also":[205],"reveals":[206],"focus":[208],"practice":[212],"on":[213],"few":[215],"criteria.":[216],"Beyond":[217],"immediate":[218],"usefulness,":[219],"researchers":[221],"urged":[223],"investigate":[225],"ways":[226],"long-term":[230],"organizational":[231],"impact":[232,236],"societal":[235]},"counts_by_year":[{"year":2026,"cited_by_count":11},{"year":2025,"cited_by_count":32},{"year":2024,"cited_by_count":21},{"year":2023,"cited_by_count":41},{"year":2022,"cited_by_count":23},{"year":2021,"cited_by_count":29},{"year":2020,"cited_by_count":31},{"year":2019,"cited_by_count":19},{"year":2018,"cited_by_count":22},{"year":2017,"cited_by_count":11},{"year":2016,"cited_by_count":4},{"year":2015,"cited_by_count":1}],"updated_date":"2026-03-29T08:15:47.926485","created_date":"2016-06-24T00:00:00"}