{"id":"https://openalex.org/W4288057714","doi":"https://doi.org/10.1109/sp46214.2022.9833581","title":"\u201cFlawed, but like democracy we don\u2019t have a better system\u201d: The Experts\u2019 Insights on the Peer Review Process of Evaluating Security Papers","display_name":"\u201cFlawed, but like democracy we don\u2019t have a better system\u201d: The Experts\u2019 Insights on the Peer Review Process of Evaluating Security Papers","publication_year":2022,"publication_date":"2022-05-01","ids":{"openalex":"https://openalex.org/W4288057714","doi":"https://doi.org/10.1109/sp46214.2022.9833581"},"language":"en","primary_location":{"id":"doi:10.1109/sp46214.2022.9833581","is_oa":false,"landing_page_url":"https://doi.org/10.1109/sp46214.2022.9833581","pdf_url":null,"source":{"id":"https://openalex.org/S4363606603","display_name":"2022 IEEE Symposium on Security and Privacy (SP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE Symposium on Security and Privacy (SP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089789807","display_name":"Ananta Soneji","orcid":"https://orcid.org/0000-0002-3045-5183"},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]},{"id":"https://openalex.org/I96749437","display_name":"Texas A&M University \u2013 Corpus Christi","ror":"https://ror.org/01mrfdz82","country_code":"US","type":"education","lineage":["https://openalex.org/I96749437"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ananta Soneji","raw_affiliation_strings":["Arizona State University,Corpus Christi","Arizona State University, Corpus Christi"],"affiliations":[{"raw_affiliation_string":"Arizona State University,Corpus Christi","institution_ids":["https://openalex.org/I96749437"]},{"raw_affiliation_string":"Arizona State University, Corpus Christi","institution_ids":["https://openalex.org/I96749437","https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000008642","display_name":"Faris Bugra Kokulu","orcid":null},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]},{"id":"https://openalex.org/I96749437","display_name":"Texas A&M University \u2013 Corpus Christi","ror":"https://ror.org/01mrfdz82","country_code":"US","type":"education","lineage":["https://openalex.org/I96749437"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Faris Bugra Kokulu","raw_affiliation_strings":["Arizona State University,Corpus Christi","Arizona State University, Corpus Christi"],"affiliations":[{"raw_affiliation_string":"Arizona State University,Corpus Christi","institution_ids":["https://openalex.org/I96749437"]},{"raw_affiliation_string":"Arizona State University, Corpus Christi","institution_ids":["https://openalex.org/I96749437","https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031836695","display_name":"Carlos Rubio-Medrano","orcid":"https://orcid.org/0000-0001-8931-6412"},"institutions":[{"id":"https://openalex.org/I96749437","display_name":"Texas A&M University \u2013 Corpus Christi","ror":"https://ror.org/01mrfdz82","country_code":"US","type":"education","lineage":["https://openalex.org/I96749437"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Carlos Rubio-Medrano","raw_affiliation_strings":["Texas A&M University,Corpus Christi"],"affiliations":[{"raw_affiliation_string":"Texas A&M University,Corpus Christi","institution_ids":["https://openalex.org/I96749437"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076987446","display_name":"Tiffany Bao","orcid":"https://orcid.org/0000-0001-6424-0001"},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]},{"id":"https://openalex.org/I96749437","display_name":"Texas A&M University \u2013 Corpus Christi","ror":"https://ror.org/01mrfdz82","country_code":"US","type":"education","lineage":["https://openalex.org/I96749437"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tiffany Bao","raw_affiliation_strings":["Arizona State University,Corpus Christi","Arizona State University, Corpus Christi"],"affiliations":[{"raw_affiliation_string":"Arizona State University,Corpus Christi","institution_ids":["https://openalex.org/I96749437"]},{"raw_affiliation_string":"Arizona State University, Corpus Christi","institution_ids":["https://openalex.org/I96749437","https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100653711","display_name":"Ruoyu Wang","orcid":"https://orcid.org/0000-0002-7240-558X"},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]},{"id":"https://openalex.org/I96749437","display_name":"Texas A&M University \u2013 Corpus Christi","ror":"https://ror.org/01mrfdz82","country_code":"US","type":"education","lineage":["https://openalex.org/I96749437"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ruoyu Wang","raw_affiliation_strings":["Arizona State University,Corpus Christi","Arizona State University, Corpus Christi"],"affiliations":[{"raw_affiliation_string":"Arizona State University,Corpus Christi","institution_ids":["https://openalex.org/I96749437"]},{"raw_affiliation_string":"Arizona State University, Corpus Christi","institution_ids":["https://openalex.org/I96749437","https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026842092","display_name":"Yan Shoshitaishvili","orcid":"https://orcid.org/0000-0001-8832-1789"},"institutions":[{"id":"https://openalex.org/I96749437","display_name":"Texas A&M University \u2013 Corpus Christi","ror":"https://ror.org/01mrfdz82","country_code":"US","type":"education","lineage":["https://openalex.org/I96749437"]},{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yan Shoshitaishvili","raw_affiliation_strings":["Arizona State University,Corpus Christi","Arizona State University, Corpus Christi"],"affiliations":[{"raw_affiliation_string":"Arizona State University,Corpus Christi","institution_ids":["https://openalex.org/I96749437"]},{"raw_affiliation_string":"Arizona State University, Corpus Christi","institution_ids":["https://openalex.org/I96749437","https://openalex.org/I55732556"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5050806439","display_name":"Adam Doup\u00e9","orcid":"https://orcid.org/0000-0003-2634-3901"},"institutions":[{"id":"https://openalex.org/I96749437","display_name":"Texas A&M University \u2013 Corpus Christi","ror":"https://ror.org/01mrfdz82","country_code":"US","type":"education","lineage":["https://openalex.org/I96749437"]},{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Adam Doup\u00e9","raw_affiliation_strings":["Arizona State University,Corpus Christi","Arizona State University, Corpus Christi"],"affiliations":[{"raw_affiliation_string":"Arizona State University,Corpus Christi","institution_ids":["https://openalex.org/I96749437"]},{"raw_affiliation_string":"Arizona State University, Corpus Christi","institution_ids":["https://openalex.org/I96749437","https://openalex.org/I55732556"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5089789807"],"corresponding_institution_ids":["https://openalex.org/I55732556","https://openalex.org/I96749437"],"apc_list":null,"apc_paid":null,"fwci":0.7301,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.72029172,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1845","last_page":"1862"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10734","display_name":"Information and Cyber Security","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12519","display_name":"Cybercrime and Law Enforcement Studies","score":0.9886999726295471,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/peer-review","display_name":"Peer review","score":0.6402590274810791},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.608919084072113},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5907077193260193},{"id":"https://openalex.org/keywords/publishing","display_name":"Publishing","score":0.5704860687255859},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.49959230422973633},{"id":"https://openalex.org/keywords/public-relations","display_name":"Public relations","score":0.45991411805152893},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.40812957286834717},{"id":"https://openalex.org/keywords/internet-privacy","display_name":"Internet privacy","score":0.40137752890586853},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.3382404148578644},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.2693999707698822},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.12227368354797363}],"concepts":[{"id":"https://openalex.org/C138368954","wikidata":"https://www.wikidata.org/wiki/Q215028","display_name":"Peer review","level":2,"score":0.6402590274810791},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.608919084072113},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5907077193260193},{"id":"https://openalex.org/C151719136","wikidata":"https://www.wikidata.org/wiki/Q3972943","display_name":"Publishing","level":2,"score":0.5704860687255859},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.49959230422973633},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.45991411805152893},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.40812957286834717},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.40137752890586853},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3382404148578644},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.2693999707698822},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.12227368354797363},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/sp46214.2022.9833581","is_oa":false,"landing_page_url":"https://doi.org/10.1109/sp46214.2022.9833581","pdf_url":null,"source":{"id":"https://openalex.org/S4363606603","display_name":"2022 IEEE Symposium on Security and Privacy (SP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 IEEE Symposium on Security and Privacy (SP)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.44999998807907104,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332815","display_name":"Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":55,"referenced_works":["https://openalex.org/W1541396790","https://openalex.org/W1780778418","https://openalex.org/W1981449122","https://openalex.org/W1989364426","https://openalex.org/W1991330531","https://openalex.org/W1999365717","https://openalex.org/W2001356016","https://openalex.org/W2011369055","https://openalex.org/W2017899036","https://openalex.org/W2026674089","https://openalex.org/W2041212401","https://openalex.org/W2043948277","https://openalex.org/W2047081701","https://openalex.org/W2047294926","https://openalex.org/W2051361469","https://openalex.org/W2068754596","https://openalex.org/W2121257406","https://openalex.org/W2130500927","https://openalex.org/W2142168034","https://openalex.org/W2164777277","https://openalex.org/W2165701842","https://openalex.org/W2297737328","https://openalex.org/W2679033717","https://openalex.org/W2750586499","https://openalex.org/W2752524112","https://openalex.org/W2754461217","https://openalex.org/W2766913883","https://openalex.org/W2783459865","https://openalex.org/W2796036468","https://openalex.org/W2801023641","https://openalex.org/W2811088837","https://openalex.org/W2895375884","https://openalex.org/W2917732241","https://openalex.org/W2949172052","https://openalex.org/W2998216072","https://openalex.org/W3006328987","https://openalex.org/W3029793458","https://openalex.org/W3039299463","https://openalex.org/W3042703261","https://openalex.org/W3127588629","https://openalex.org/W3157068929","https://openalex.org/W3157464625","https://openalex.org/W3158201466","https://openalex.org/W3159884981","https://openalex.org/W3194023449","https://openalex.org/W4221032546","https://openalex.org/W4241559800","https://openalex.org/W4300858789","https://openalex.org/W6743839099","https://openalex.org/W6780393192","https://openalex.org/W6790487486","https://openalex.org/W6794343756","https://openalex.org/W6794922378","https://openalex.org/W6794999304","https://openalex.org/W6991547700"],"related_works":["https://openalex.org/W2294677930","https://openalex.org/W2801905285","https://openalex.org/W2028846973","https://openalex.org/W2698935035","https://openalex.org/W2756635080","https://openalex.org/W249766449","https://openalex.org/W1504443673","https://openalex.org/W2230419033","https://openalex.org/W2086821603","https://openalex.org/W2354149895"],"abstract_inverted_index":{"The":[0],"academic":[1],"computer":[2,77],"security":[3,78,101,110,125,190],"community":[4,41,136,183],"has":[5],"traditionally":[6],"adopted":[7],"peer":[8,72,111,191],"review":[9,73,112,126,192],"as":[10],"an":[11],"integral":[12],"part":[13],"of":[14,47,59,156,173,182],"scientific":[15],"publishing":[16],"and":[17,25,31,37,103,150],"dissemination,":[18],"in":[19,52,75,99,123,140,166],"a":[20,45,171],"process":[21,51,74],"that":[22,185],"grows":[23],"organically":[24],"nourishes":[26],"itself":[27],"by":[28,134],"internal":[29],"communications":[30],"intuitions,":[32],"rather":[33],"than":[34],"repeatable":[35],"experiments":[36],"investigations.":[38],"Recently,":[39],"key":[40],"members":[42],"have":[43,130,146,159],"shared":[44],"series":[46,172],"concerns":[48,106],"regarding":[49,107],"this":[50,62],"public.":[53],"To":[54],"support":[55],"or":[56],"disprove":[57],"some":[58,129],"these":[60],"concerns,":[61],"paper":[63],"presents":[64],"the":[65,71,76,91,108,116,124,135,138,154,163,179,189],"first":[66,164],"qualitative":[67],"study":[68],"to":[69,177],"examine":[70],"field.":[79],"Through":[80],"semi-structured":[81],"interviews":[82],"(n=21)":[83],"with":[84],"Program":[85],"Committee":[86],"members,":[87],"we":[88,119,175],"systematically":[89],"collect":[90],"reviewers\u2019":[92,141],"insights":[93],"on":[94,115,153],"how":[95],"papers":[96],"are":[97],"evaluated":[98],"top-tier":[100],"conferences":[102],"investigate":[104],"their":[105],"current":[109],"system.":[113],"Based":[114],"collected":[117],"data,":[118],"identify":[120],"several":[121],"issues":[122],"system:":[127],"whereas":[128],"been":[131,160],"previously":[132],"observed":[133,161],"(e.g.,":[137,144],"randomness":[139],"decisions),":[142],"others":[143],"reviewers":[145],"much":[147],"more":[148],"diverse":[149],"concrete":[151],"opinions":[152],"metrics":[155],"rejecting":[157],"papers)":[158],"for":[162],"time":[165],"our":[167],"study.":[168],"Finally,":[169],"through":[170],"recommendations,":[174],"aim":[176],"encourage":[178],"collaborative":[180],"establishment":[181],"norms":[184],"will":[186],"significantly":[187],"improve":[188],"process.":[193]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1}],"updated_date":"2026-04-01T17:29:45.350535","created_date":"2025-10-10T00:00:00"}