{"id":"https://openalex.org/W2407772516","doi":"https://doi.org/10.1145/2915970.2916008","title":"The need for multivocal literature reviews in software engineering","display_name":"The need for multivocal literature reviews in software engineering","publication_year":2016,"publication_date":"2016-05-24","ids":{"openalex":"https://openalex.org/W2407772516","doi":"https://doi.org/10.1145/2915970.2916008","mag":"2407772516"},"language":"en","primary_location":{"id":"doi:10.1145/2915970.2916008","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2915970.2916008","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 20th International Conference on Evaluation and Assessment in Software Engineering","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000712015","display_name":"Vahid Garousi","orcid":"https://orcid.org/0000-0001-6590-7576"},"institutions":[{"id":"https://openalex.org/I66514158","display_name":"Hacettepe University","ror":"https://ror.org/04kwvgz42","country_code":"TR","type":"education","lineage":["https://openalex.org/I66514158"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Vahid Garousi","raw_affiliation_strings":["Hacettepe University, Ankara, Turkey"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hacettepe University, Ankara, Turkey","institution_ids":["https://openalex.org/I66514158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055476405","display_name":"Michael Felderer","orcid":"https://orcid.org/0000-0003-3818-4442"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Michael Felderer","raw_affiliation_strings":["University of Innsbruck, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Innsbruck, Austria","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078824435","display_name":"Mika M\u00e4ntyl\u00e4","orcid":"https://orcid.org/0000-0002-2841-5879"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Mika V. M\u00e4ntyl\u00e4","raw_affiliation_strings":["University of Oulu, Oulu, Finland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oulu, Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":209,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/grey-literature","display_name":"Grey literature","score":0.7361248731613159},{"id":"https://openalex.org/keywords/viewpoints","display_name":"Viewpoints","score":0.5773552656173706},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.4776533842086792},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4211653769016266},{"id":"https://openalex.org/keywords/library-science","display_name":"Library science","score":0.36664488911628723},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.1685141623020172},{"id":"https://openalex.org/keywords/medline","display_name":"MEDLINE","score":0.08028009533882141},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.06820613145828247}],"concepts":[{"id":"https://openalex.org/C61904560","wikidata":"https://www.wikidata.org/wiki/Q1148359","display_name":"Grey literature","level":3,"score":0.7361248731613159},{"id":"https://openalex.org/C2776035091","wikidata":"https://www.wikidata.org/wiki/Q7928819","display_name":"Viewpoints","level":2,"score":0.5773552656173706},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.4776533842086792},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4211653769016266},{"id":"https://openalex.org/C161191863","wikidata":"https://www.wikidata.org/wiki/Q199655","display_name":"Library science","level":1,"score":0.36664488911628723},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.1685141623020172},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.08028009533882141},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.06820613145828247},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2915970.2916008","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2915970.2916008","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 20th International Conference on Evaluation and Assessment in Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","score":0.7200000286102295,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W18855373","https://openalex.org/W1246473807","https://openalex.org/W1493688518","https://openalex.org/W1512902557","https://openalex.org/W1558527408","https://openalex.org/W1563138110","https://openalex.org/W1862376005","https://openalex.org/W1898915641","https://openalex.org/W1911705068","https://openalex.org/W1972888971","https://openalex.org/W1992602600","https://openalex.org/W1999798506","https://openalex.org/W2008155708","https://openalex.org/W2025183119","https://openalex.org/W2028836139","https://openalex.org/W2033570995","https://openalex.org/W2037316754","https://openalex.org/W2092245922","https://openalex.org/W2098741004","https://openalex.org/W2101189852","https://openalex.org/W2106956101","https://openalex.org/W2108025693","https://openalex.org/W2138571267","https://openalex.org/W2202602917","https://openalex.org/W2231868335","https://openalex.org/W2233713202","https://openalex.org/W2250631455","https://openalex.org/W2310208102","https://openalex.org/W2399281224","https://openalex.org/W2487033419","https://openalex.org/W2528774446","https://openalex.org/W2955867509","https://openalex.org/W3021836553","https://openalex.org/W3207900863","https://openalex.org/W4229772528","https://openalex.org/W4240847688","https://openalex.org/W4302078553","https://openalex.org/W6689502610"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2385368906","https://openalex.org/W2902924992","https://openalex.org/W2626642044","https://openalex.org/W4250451092","https://openalex.org/W1911705068","https://openalex.org/W2336383631","https://openalex.org/W1544537990","https://openalex.org/W2984938923","https://openalex.org/W2216704622"],"abstract_inverted_index":{"Systematic":[0],"Literature":[1,28],"Reviews":[2],"(SLR)":[3],"may":[4],"not":[5,19,86],"provide":[6],"insight":[7],"into":[8],"the":[9,12,22,44,63,88,107,127,130,138,159,182],"\"state":[10],"of":[11,34,78,154,161],"practice\"":[13],"in":[14,41,55,67,91,151,170,193],"SE,":[15,155],"as":[16,100,168],"they":[17],"do":[18,98],"typically":[20],"include":[21,87,106],"\"grey\"":[23],"(non-published)":[24],"literature.":[25,47,140],"A":[26],"Multivocal":[27],"Review":[29],"(MLR)":[30],"is":[31,172],"a":[32,35,49,83,92,101,119],"form":[33],"SLR":[36,84],"which":[37],"includes":[38],"grey":[39,139,145,162],"literature":[40,90,109,146,163],"addition":[42],"to":[43,134,186],"published":[45,54],"(formal)":[46],"Only":[48],"few":[50,120],"MLRs":[51,66,124,192],"have":[52],"been":[53],"SE":[56,68,93],"so":[57],"far.":[58],"We":[59,141],"aim":[60],"at":[61],"raising":[62],"awareness":[64],"for":[65,190],"by":[69],"addressing":[70],"two":[71],"research":[72],"questions":[73],"(RQs):":[74],"(1)":[75,144],"What":[76,97],"types":[77],"knowledge":[79,132],"are":[80,184],"missed":[81],"when":[82,104],"does":[85],"multivocal":[89,108],"field?":[94],"and":[95,110,123,125,129,156,175],"(2)":[96,158],"we,":[99],"community,":[102],"gain":[103],"we":[105,117],"conduct":[111],"MLRs?":[112],"To":[113],"answer":[114],"these":[115,179],"RQs,":[116],"sample":[118],"example":[121],"SLRs":[122],"identify":[126],"missing":[128],"gained":[131],"due":[133],"excluding":[135],"or":[136],"including":[137],"find":[142],"that":[143,157],"can":[147],"give":[148],"substantial":[149],"benefits":[150],"certain":[152,166],"areas":[153],"inclusion":[160],"brings":[164],"forward":[165],"challenges":[167],"evidence":[169],"them":[171],"often":[173],"experience":[174],"opinion":[176],"based.":[177],"Given":[178],"conflicting":[180],"viewpoints,":[181],"authors":[183],"planning":[185],"prepare":[187],"systematic":[188],"guidelines":[189],"performing":[191],"SE.":[194]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":20},{"year":2024,"cited_by_count":17},{"year":2023,"cited_by_count":15},{"year":2022,"cited_by_count":24},{"year":2021,"cited_by_count":29},{"year":2020,"cited_by_count":26},{"year":2019,"cited_by_count":21},{"year":2018,"cited_by_count":35},{"year":2017,"cited_by_count":13},{"year":2016,"cited_by_count":6}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}