{"id":"https://openalex.org/W4226134286","doi":"https://doi.org/10.1109/tse.2022.3165938","title":"How Should Software Engineering Secondary Studies Include Grey Material?","display_name":"How Should Software Engineering Secondary Studies Include Grey Material?","publication_year":2022,"publication_date":"2022-04-08","ids":{"openalex":"https://openalex.org/W4226134286","doi":"https://doi.org/10.1109/tse.2022.3165938"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2022.3165938","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tse.2022.3165938","pdf_url":"https://ieeexplore.ieee.org/ielx7/32/10044379/09754223.pdf","source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://ieeexplore.ieee.org/ielx7/32/10044379/09754223.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5012102325","display_name":"Barbara Kitchenham","orcid":"https://orcid.org/0000-0002-6134-8460"},"institutions":[{"id":"https://openalex.org/I56007636","display_name":"Keele University","ror":"https://ror.org/00340yn33","country_code":"GB","type":"education","lineage":["https://openalex.org/I56007636"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Barbara Kitchenham","raw_affiliation_strings":["School of Computing and Mathematics, Keele University, Keele, Staffordshire, U.K"],"affiliations":[{"raw_affiliation_string":"School of Computing and Mathematics, Keele University, Keele, Staffordshire, U.K","institution_ids":["https://openalex.org/I56007636"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081451369","display_name":"Lech Madeyski","orcid":"https://orcid.org/0000-0003-3907-3357"},"institutions":[{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]},{"id":"https://openalex.org/I11923345","display_name":"Wroc\u0142aw University of Science and Technology","ror":"https://ror.org/008fyn775","country_code":"PL","type":"education","lineage":["https://openalex.org/I11923345"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Lech Madeyski","raw_affiliation_strings":["Department of Applied Informatics, Wroclaw University of Science and Technology, Wroclaw, Poland"],"affiliations":[{"raw_affiliation_string":"Department of Applied Informatics, Wroclaw University of Science and Technology, Wroclaw, Poland","institution_ids":["https://openalex.org/I11923345","https://openalex.org/I686019"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061755008","display_name":"David Budgen","orcid":"https://orcid.org/0000-0001-7143-0241"},"institutions":[{"id":"https://openalex.org/I190082696","display_name":"Durham University","ror":"https://ror.org/01v29qb04","country_code":"GB","type":"education","lineage":["https://openalex.org/I190082696"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"David Budgen","raw_affiliation_strings":["Department of Computer Science, Durham University, Durham, U.K"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Durham University, Durham, U.K","institution_ids":["https://openalex.org/I190082696"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5012102325"],"corresponding_institution_ids":["https://openalex.org/I56007636"],"apc_list":null,"apc_paid":null,"fwci":5.729,"has_fulltext":true,"cited_by_count":20,"citation_normalized_percentile":{"value":0.96170337,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"49","issue":"2","first_page":"872","last_page":"882"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9422000050544739,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5909836888313293},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5596531629562378},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.45565083622932434},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.4392128884792328},{"id":"https://openalex.org/keywords/library-science","display_name":"Library science","score":0.4304393529891968},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.15884879231452942},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.130859375}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5909836888313293},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5596531629562378},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.45565083622932434},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4392128884792328},{"id":"https://openalex.org/C161191863","wikidata":"https://www.wikidata.org/wiki/Q199655","display_name":"Library science","level":1,"score":0.4304393529891968},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.15884879231452942},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.130859375},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tse.2022.3165938","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tse.2022.3165938","pdf_url":"https://ieeexplore.ieee.org/ielx7/32/10044379/09754223.pdf","source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:dro.dur.ac.uk.OAI2:35647","is_oa":true,"landing_page_url":"http://dro.dur.ac.uk/35647/","pdf_url":"https://durham-repository.worktribe.com/preview/1210060/35647VoR.pdf","source":{"id":"https://openalex.org/S4377196258","display_name":"Durham Research Online (Durham University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I190082696","host_organization_name":"Durham University","host_organization_lineage":["https://openalex.org/I190082696"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions on Software Engineering, 2023, Vol.49(2), pp.872-882 [Peer Reviewed Journal]","raw_type":"Article"},{"id":"pmh:oai:eprints.keele.ac.uk:12133","is_oa":true,"landing_page_url":"https://eprints.keele.ac.uk/id/eprint/12133/1/Kitchenham-2023-TSE-GreyLiterature.pdf","pdf_url":"https://keele-repository.worktribe.com/preview/954759/Supervising%20criminalised%20women%20in%20the%20community%20in%20England%20and%20Wales-%20A%20brief%20history%20of%20%28un%29changing%20attitudes%20-%20Vanstone%20-%20The%20Howard%20Journal%20of%20Crime%20and%20Justice%20-%20Wiley%20Online%20Library.pdf","source":{"id":"https://openalex.org/S4306402149","display_name":"Keele Research Repository (Keele University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I56007636","host_organization_name":"Keele University","host_organization_lineage":["https://openalex.org/I56007636"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"},{"id":"pmh:oai:durham-repository.worktribe.com:1210041","is_oa":true,"landing_page_url":"https://durham-repository.worktribe.com/output/1210041","pdf_url":null,"source":{"id":"https://openalex.org/S4306400188","display_name":"Durham Research Online (Durham University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I190082696","host_organization_name":"Durham University","host_organization_lineage":["https://openalex.org/I190082696"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Journal Article"}],"best_oa_location":{"id":"doi:10.1109/tse.2022.3165938","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tse.2022.3165938","pdf_url":"https://ieeexplore.ieee.org/ielx7/32/10044379/09754223.pdf","source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4226134286.pdf","grobid_xml":"https://content.openalex.org/works/W4226134286.grobid-xml"},"referenced_works_count":47,"referenced_works":["https://openalex.org/W1946891069","https://openalex.org/W1986736724","https://openalex.org/W1990176269","https://openalex.org/W2006479099","https://openalex.org/W2027102540","https://openalex.org/W2045571829","https://openalex.org/W2050059866","https://openalex.org/W2070300937","https://openalex.org/W2118283821","https://openalex.org/W2129646044","https://openalex.org/W2139357103","https://openalex.org/W2163851162","https://openalex.org/W2203490966","https://openalex.org/W2282619714","https://openalex.org/W2336383631","https://openalex.org/W2345039647","https://openalex.org/W2380271241","https://openalex.org/W2407772516","https://openalex.org/W2489347314","https://openalex.org/W2508327075","https://openalex.org/W2525353023","https://openalex.org/W2567362164","https://openalex.org/W2576494997","https://openalex.org/W2600366513","https://openalex.org/W2612764117","https://openalex.org/W2735864825","https://openalex.org/W2766446162","https://openalex.org/W2774161397","https://openalex.org/W2775212908","https://openalex.org/W2790894500","https://openalex.org/W2795866244","https://openalex.org/W2908082161","https://openalex.org/W2936534160","https://openalex.org/W2990737658","https://openalex.org/W3004454911","https://openalex.org/W3005586235","https://openalex.org/W3010620368","https://openalex.org/W3011193568","https://openalex.org/W3085010947","https://openalex.org/W3091407142","https://openalex.org/W3150615609","https://openalex.org/W3170490302","https://openalex.org/W4247781538","https://openalex.org/W4253314500","https://openalex.org/W6687875428","https://openalex.org/W6722705644","https://openalex.org/W6784422159"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2478288626","https://openalex.org/W2350741829","https://openalex.org/W2530322880","https://openalex.org/W1596801655"],"abstract_inverted_index":{"<italic":[0,11,48,68,132,228,233,317],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[1,12,49,69,133,229,234,318],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Context</i>":[2],":":[3,51,71,135,320],"Recent":[4],"papers":[5,20],"have":[6,21,138,150],"proposed":[7],"the":[8,25,44,74,79,98,104,110,126,159,163,212,220,227,232,263,279,310,369],"use":[9,127,201,365],"of":[10,76,81,84,94,106,112,128,146,154,165,202,214,236,304,312,372],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">grey":[13],"literature</i>":[14],"(GL)":[15],"and":[16,36,65,78,83,118,120,149,307,385],"multivocal":[17,66],"reviews.":[18,67],"These":[19],"raised":[22],"issues":[23],"about":[24],"practices":[26],"used":[27,100,285,308],"for":[28,216,323,341],"systematic":[29],"reviews":[30],"(SRs)":[31],"in":[32,88,101,130,258,278,290,302,309],"software":[33,314],"engineering":[34,315],"(SE)":[35],"suggested":[37],"that":[38,96,157,174,195,238,270],"there":[39],"should":[40,282,352],"be":[41,60,115,272,275,284,334,353],"changes":[42],"to":[43,59,62,114,170,172,183,274,333,374,386],"current":[45,55],"SR":[46,56,136,218,260,264],"guidelines.":[47,361],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Objective</i>":[50],"To":[52],"investigate":[53],"whether":[54],"guidelines":[57,137,322,340],"need":[58],"changed":[61],"support":[63],"GL":[64,77,82,141,203],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Method</i>":[70],"We":[72,91],"discuss":[73],"definitions":[75],"importance":[80],"industry-based":[85],"field":[86,155],"studies":[87,148,156],"SE":[89,337,362],"SRs.":[90,131],"identify":[92],"properties":[93],"SRs":[95,113,196,324,367,384],"constrain":[97],"material":[99],"SRs:":[102],"a)":[103],"nature":[105],"primary":[107,147,221,251,288,357],"studies;":[108],"b)":[109],"requirements":[111,124],"auditable,":[116],"traceable,":[117],"reproducible;":[119],"explain":[121],"why":[122],"these":[123,351],"restrict":[125],"blogs":[129],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Results</i>":[134],"always":[139],"considered":[140],"as":[142,188,286],"a":[143,244,249,287],"possible":[144],"source":[145],"never":[151],"supported":[152],"exclusion":[153],"incorporate":[158,375],"practitioners\u2019":[160],"viewpoint.":[161],"However,":[162,211,267],"concept":[164],"GL,":[166],"which":[167],"was":[168],"meant":[169],"refer":[171],"documents":[173],"were":[175],"not":[176,198,207,226,283,295,326],"formally":[177],"published,":[178],"is":[179,219,225,239,248,255],"now":[180],"being":[181,300],"extended":[182],"information":[184,237,298],"from":[185,299,377,383],"sources":[186],"such":[187,209,297],"blogs/tweets/Q&A":[189],"posts.":[190],"Thus,":[191,223],"it":[192,224,254],"might":[193],"seem":[194],"do":[197,206,325],"make":[199],"full":[200],"because":[204],"they":[205],"include":[208],"information.":[210],"unit":[213],"analysis":[215],"an":[217,259,291],"study.":[222,252],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">source</i>":[230],"but":[231,329,350],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">type</i>":[235],"important.":[240],"Any":[241],"report":[242],"describing":[243],"rigorous":[245],"empirical":[246],"evaluation":[247],"candidate":[250],"Whether":[253],"actually":[256],"included":[257],"depends":[261],"on":[262,355],"eligibility":[265],"criteria.":[266],"any":[268],"study":[269,289,360],"cannot":[271],"guaranteed":[273],"publicly":[276],"available":[277],"long":[280],"term":[281],"SR.":[292],"This":[293],"does":[294],"prevent":[296],"aggregated":[301],"surveys":[303,380],"social":[305,343,378],"media":[306,344,379],"context":[311],"evidence-based":[313],"(EBSE).":[316],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Conclusions</i>":[319],"Current":[321],"require":[327,339],"extensions,":[328],"their":[330],"scope":[331],"needs":[332],"better":[335],"defined.":[336],"researchers":[338,363],"analysing":[342],"posts":[345],"(e.g.,":[346],"blogs,":[347],"tweets,":[348],"vlogs),":[349],"based":[354],"qualitative":[356],"(not":[358],"secondary)":[359],"can":[364],"mixed-methods":[366],"and/or":[368],"fourth":[370],"step":[371],"EBSE":[373],"findings":[376],"with":[381],"those":[382],"develop":[387],"industry-relevant":[388],"recommendations.":[389]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}