{"id":"https://openalex.org/W3202958116","doi":"https://doi.org/10.1145/3475716.3475777","title":"What Evidence We Would Miss If We Do Not Use Grey Literature?","display_name":"What Evidence We Would Miss If We Do Not Use Grey Literature?","publication_year":2021,"publication_date":"2021-10-06","ids":{"openalex":"https://openalex.org/W3202958116","doi":"https://doi.org/10.1145/3475716.3475777","mag":"3202958116"},"language":"en","primary_location":{"id":"doi:10.1145/3475716.3475777","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3475716.3475777","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th ACM / IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5044929087","display_name":"Fernando Kamei","orcid":"https://orcid.org/0000-0002-5572-2049"},"institutions":[{"id":"https://openalex.org/I4210155484","display_name":"Instituto Federal de Educa\u00e7\u00e3o, Ci\u00eancia e Tecnologia de Alagoas","ror":"https://ror.org/040ys9e84","country_code":"BR","type":"education","lineage":["https://openalex.org/I4210155484"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Fernando Kamei","raw_affiliation_strings":["UFPE, IFAL, Macei\u00f3, Alagoas, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UFPE, IFAL, Macei\u00f3, Alagoas, Brazil","institution_ids":["https://openalex.org/I4210155484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016508154","display_name":"Gustavo Pinto","orcid":"https://orcid.org/0000-0001-7598-2799"},"institutions":[{"id":"https://openalex.org/I59606676","display_name":"Universidade Federal do Par\u00e1","ror":"https://ror.org/03q9sr818","country_code":"BR","type":"education","lineage":["https://openalex.org/I59606676"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Gustavo Pinto","raw_affiliation_strings":["UFPA, Bel\u00e9m, Par\u00e1, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UFPA, Bel\u00e9m, Par\u00e1, Brazil","institution_ids":["https://openalex.org/I59606676"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060903936","display_name":"Igor Wiese","orcid":"https://orcid.org/0000-0001-9943-5570"},"institutions":[{"id":"https://openalex.org/I1283613182","display_name":"Universidade Tecnol\u00f3gica Federal do Paran\u00e1","ror":"https://ror.org/002v2kq79","country_code":"BR","type":"education","lineage":["https://openalex.org/I1283613182"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Igor Wiese","raw_affiliation_strings":["UTFPR, Campo Mour\u00e3o, Paran\u00e1, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UTFPR, Campo Mour\u00e3o, Paran\u00e1, Brazil","institution_ids":["https://openalex.org/I1283613182"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051287042","display_name":"M\u00e1rcio Ribeiro","orcid":"https://orcid.org/0000-0002-4293-4261"},"institutions":[{"id":"https://openalex.org/I79889768","display_name":"Universidade Federal de Alagoas","ror":"https://ror.org/00dna7t83","country_code":"BR","type":"education","lineage":["https://openalex.org/I79889768"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"M\u00e1rcio Ribeiro","raw_affiliation_strings":["UFAL, Macei\u00f3, Alagoas, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UFAL, Macei\u00f3, Alagoas, Brazil","institution_ids":["https://openalex.org/I79889768"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029015672","display_name":"S\u00e9rgio Soares","orcid":"https://orcid.org/0000-0002-4428-2535"},"institutions":[{"id":"https://openalex.org/I4210138879","display_name":"Hospital das Cl\u00ednicas da Universidade Federal de Pernambuco","ror":"https://ror.org/03rehvw54","country_code":"BR","type":"healthcare","lineage":["https://openalex.org/I4210138879"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"S\u00e9rgio Soares","raw_affiliation_strings":["UFPE, Recife, Pernambuco, Brazil"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"UFPE, Recife, Pernambuco, Brazil","institution_ids":["https://openalex.org/I4210138879"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.0514,"has_fulltext":false,"cited_by_count":21,"citation_normalized_percentile":{"value":0.89611427,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.8745999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.8745999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.8528000116348267,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.8216000199317932,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6898590326309204},{"id":"https://openalex.org/keywords/grey-literature","display_name":"Grey literature","score":0.6503664255142212},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.45073428750038147},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.4454166889190674},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.1448722779750824},{"id":"https://openalex.org/keywords/history","display_name":"History","score":0.1308152973651886},{"id":"https://openalex.org/keywords/medline","display_name":"MEDLINE","score":0.1305069923400879}],"concepts":[{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6898590326309204},{"id":"https://openalex.org/C61904560","wikidata":"https://www.wikidata.org/wiki/Q1148359","display_name":"Grey literature","level":3,"score":0.6503664255142212},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.45073428750038147},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.4454166889190674},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.1448722779750824},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.1308152973651886},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.1305069923400879},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3475716.3475777","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3475716.3475777","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th ACM / IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7699999809265137}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":24,"referenced_works":["https://openalex.org/W1562463929","https://openalex.org/W1979290264","https://openalex.org/W2095057899","https://openalex.org/W2407772516","https://openalex.org/W2620872098","https://openalex.org/W2623177513","https://openalex.org/W2735864825","https://openalex.org/W2772141859","https://openalex.org/W2795150358","https://openalex.org/W2810377871","https://openalex.org/W2899274966","https://openalex.org/W2903596337","https://openalex.org/W2908082161","https://openalex.org/W2946023400","https://openalex.org/W2955940099","https://openalex.org/W2968632928","https://openalex.org/W2978020686","https://openalex.org/W2985276741","https://openalex.org/W3004454911","https://openalex.org/W3008438078","https://openalex.org/W3091407142","https://openalex.org/W3106532184","https://openalex.org/W3114035905","https://openalex.org/W4234473732"],"related_works":["https://openalex.org/W1911705068","https://openalex.org/W2751650523","https://openalex.org/W2008155708","https://openalex.org/W2336383631","https://openalex.org/W2145616864","https://openalex.org/W1246473807","https://openalex.org/W1842739385","https://openalex.org/W2205558186","https://openalex.org/W2884358209","https://openalex.org/W2131361607"],"abstract_inverted_index":{"Context:":[0],"Multivocal":[1],"Literature":[2,11,15],"Reviews":[3],"(MLR)":[4],"search":[5],"for":[6],"evidence":[7,63,115,133,188],"in":[8,21,103,136,167,204],"both":[9],"Traditional":[10],"(TL)":[12],"and":[13,91,106,155,159],"Grey":[14],"(GL).":[16],"Despite":[17],"the":[18,24,44,100,111,141,157,168,205,218],"growing":[19],"interest":[20],"MLR-based":[22],"studies,":[23],"literature":[25],"assessing":[26],"how":[27,43,189],"GL":[28,47,60,101,131,142,163,190],"has":[29],"contributed":[30,48,191],"to":[31,41,49,57,71,84,108,120,146,149,192,229],"MLR":[32,50,70,86,193],"studies":[33,87,105,119],"is":[34,61,65],"still":[35],"scarce.":[36],"Objective:":[37],"This":[38],"research":[39,74,123],"aims":[40],"assess":[42],"use":[45],"of":[46,95,140],"studies.":[51,194],"By":[52],"contributing,":[53],"we":[54],"mean,":[55],"understanding":[56],"what":[58,109],"extent":[59,110],"providing":[62,114],"that":[64,116,129,197],"indeed":[66],"used":[67,102,145,166],"by":[68,79,175],"an":[69],"answer":[72,121],"its":[73],"question.":[75],"Method:":[76],"We":[77,97,187,195,215],"start":[78],"conducting":[80,222],"a":[81,153],"tertiary":[82],"study":[83],"identify":[85],"published":[88],"between":[89],"2017":[90],"2019,":[92],"selecting":[93],"nine":[94],"them.":[96],"then":[98],"identified":[99,128],"these":[104,118,170,199],"assessed":[107],"GLs":[112,171,200],"are":[113],"help":[117,235],"their":[122],"questions.":[124],"Results:":[125],"Our":[126],"analysis":[127],"1)":[130],"provided":[132],"not":[134,202],"found":[135],"TL,":[137],"2)":[138],"most":[139],"sources":[143],"were":[144,165,172,201],"provide":[147],"recommendations":[148],"solve":[150],"problems,":[151],"explain":[152],"topic,":[154],"classify":[156],"findings,":[158],"3)":[160],"19":[161],"different":[162],"types":[164],"studies;":[169],"mainly":[173],"produced":[174],"SE":[176,237],"practitioners":[177],"(including":[178],"blog":[179],"posts,":[180],"slides":[181],"presentations,":[182],"or":[183,213],"project":[184],"descriptions).":[185],"Conclusions:":[186],"observed":[196],"if":[198],"included":[203],"MLR,":[206],"several":[207],"findings":[208],"would":[209],"have":[210],"been":[211],"omitted":[212],"weakened.":[214],"also":[216],"described":[217],"challenges":[219],"involved":[220],"when":[221],"this":[223],"investigation,":[224],"along":[225],"with":[226,231],"potential":[227],"ways":[228],"deal":[230],"them,":[232],"which":[233],"may":[234],"future":[236],"researchers.":[238]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}