{"id":"https://openalex.org/W4224319565","doi":"https://doi.org/10.1145/3485447.3512214","title":"Reproducibility and Replicability of Web\u00a0Measurement Studies","display_name":"Reproducibility and Replicability of Web\u00a0Measurement Studies","publication_year":2022,"publication_date":"2022-04-25","ids":{"openalex":"https://openalex.org/W4224319565","doi":"https://doi.org/10.1145/3485447.3512214"},"language":"en","primary_location":{"id":"doi:10.1145/3485447.3512214","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3485447.3512214","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3485447.3512214","source":{"id":"https://openalex.org/S4363608783","display_name":"Proceedings of the ACM Web Conference 2022","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Web Conference 2022","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3485447.3512214","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031244583","display_name":"Nurullah Demir","orcid":"https://orcid.org/0000-0001-8721-4412"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Nurullah Demir","raw_affiliation_strings":["Institute for Internet Security, Germany and KASTEL Security Research Labs, Karlsruhe Institute of Technology, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Internet Security, Germany and KASTEL Security Research Labs, Karlsruhe Institute of Technology, Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006781393","display_name":"Matteo Gro\u00dfe-Kampmann","orcid":"https://orcid.org/0000-0001-9127-969X"},"institutions":[{"id":"https://openalex.org/I904495901","display_name":"Ruhr University Bochum","ror":"https://ror.org/04tsk2644","country_code":"DE","type":"education","lineage":["https://openalex.org/I904495901"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Matteo Gro\u00dfe-Kampmann","raw_affiliation_strings":["AWARE7 GmbH, Germany and Ruhr University Bochum, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"AWARE7 GmbH, Germany and Ruhr University Bochum, Germany","institution_ids":["https://openalex.org/I904495901"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089427676","display_name":"Tobias Urban","orcid":"https://orcid.org/0000-0003-0908-0038"},"institutions":[{"id":"https://openalex.org/I1317578790","display_name":"Federal Office for Information Security","ror":"https://ror.org/03v7mmm26","country_code":"DE","type":"other","lineage":["https://openalex.org/I1317578790"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Tobias Urban","raw_affiliation_strings":["Institute for Internet Security, Germany and secunet Security Networks AG, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Internet Security, Germany and secunet Security Networks AG, Germany","institution_ids":["https://openalex.org/I1317578790"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040751355","display_name":"Christian Wressnegger","orcid":"https://orcid.org/0009-0007-1493-9552"},"institutions":[{"id":"https://openalex.org/I102335020","display_name":"Karlsruhe Institute of Technology","ror":"https://ror.org/04t3en479","country_code":"DE","type":"education","lineage":["https://openalex.org/I102335020","https://openalex.org/I1305996414"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Christian Wressnegger","raw_affiliation_strings":["KASTEL Security Research Labs, Karlsruhe Institute of Technology, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"KASTEL Security Research Labs, Karlsruhe Institute of Technology, Germany","institution_ids":["https://openalex.org/I102335020"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056790702","display_name":"Thorsten Holz","orcid":"https://orcid.org/0000-0002-2783-1264"},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Thorsten Holz","raw_affiliation_strings":["CISPA Helmholtz Center for Information Security, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CISPA Helmholtz Center for Information Security, Germany","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066084830","display_name":"Norbert Pohlmann","orcid":"https://orcid.org/0009-0007-6221-7327"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Norbert Pohlmann","raw_affiliation_strings":["Institute for Internet Security, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Internet Security, Germany","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":8.8878,"has_fulltext":true,"cited_by_count":39,"citation_normalized_percentile":{"value":0.98959449,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"533","last_page":"544"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9937999844551086,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10742","display_name":"Peer-to-Peer Network Technologies","score":0.9926000237464905,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7474296689033508},{"id":"https://openalex.org/keywords/crawling","display_name":"Crawling","score":0.7374027371406555},{"id":"https://openalex.org/keywords/web-crawler","display_name":"Web crawler","score":0.5575794577598572},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.5198196172714233},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5033277869224548},{"id":"https://openalex.org/keywords/web-application","display_name":"Web application","score":0.49084144830703735},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.44386643171310425},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2412649691104889},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.12471526861190796}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7474296689033508},{"id":"https://openalex.org/C100368936","wikidata":"https://www.wikidata.org/wiki/Q1411725","display_name":"Crawling","level":2,"score":0.7374027371406555},{"id":"https://openalex.org/C13743948","wikidata":"https://www.wikidata.org/wiki/Q45842","display_name":"Web crawler","level":2,"score":0.5575794577598572},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5198196172714233},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5033277869224548},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.49084144830703735},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.44386643171310425},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2412649691104889},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.12471526861190796},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C105702510","wikidata":"https://www.wikidata.org/wiki/Q514","display_name":"Anatomy","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3485447.3512214","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3485447.3512214","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3485447.3512214","source":{"id":"https://openalex.org/S4363608783","display_name":"Proceedings of the ACM Web Conference 2022","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Web Conference 2022","raw_type":"proceedings-article"},{"id":"pmh:oai:figshare.com:article/24614082","is_oa":true,"landing_page_url":"https://figshare.com/articles/conference_contribution/Reproducibility_and_Replicability_of_Web_Measurement_Studies/24614082","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference contribution"},{"id":"doi:10.60882/cispa.24614082.v1","is_oa":true,"landing_page_url":"https://doi.org/10.60882/cispa.24614082.v1","pdf_url":null,"source":{"id":"https://openalex.org/S7407050916","display_name":"CISPA Helmholtz Center","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"ConferenceProceeding"}],"best_oa_location":{"id":"doi:10.1145/3485447.3512214","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3485447.3512214","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3485447.3512214","source":{"id":"https://openalex.org/S4363608783","display_name":"Proceedings of the ACM Web Conference 2022","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Web Conference 2022","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G18682879","display_name":"EXC 2092: CyberSicherheit im Zeitalter gro\u00dfskaliger Angreifer (CASA)","funder_award_id":"390781972","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4224319565.pdf","grobid_xml":"https://content.openalex.org/works/W4224319565.grobid-xml"},"referenced_works_count":46,"referenced_works":["https://openalex.org/W1546285722","https://openalex.org/W1972524097","https://openalex.org/W1984487609","https://openalex.org/W1994795387","https://openalex.org/W2006896671","https://openalex.org/W2083183119","https://openalex.org/W2102457045","https://openalex.org/W2118226475","https://openalex.org/W2144981148","https://openalex.org/W2155409100","https://openalex.org/W2159710624","https://openalex.org/W2272516773","https://openalex.org/W2388964330","https://openalex.org/W2424402128","https://openalex.org/W2535603283","https://openalex.org/W2573945754","https://openalex.org/W2904027722","https://openalex.org/W2922166134","https://openalex.org/W2944196828","https://openalex.org/W2951914319","https://openalex.org/W2962940036","https://openalex.org/W2964241064","https://openalex.org/W2972665508","https://openalex.org/W2987577970","https://openalex.org/W3003683799","https://openalex.org/W3004148183","https://openalex.org/W3012679427","https://openalex.org/W3015392158","https://openalex.org/W3015767612","https://openalex.org/W3015863761","https://openalex.org/W3015896759","https://openalex.org/W3015975623","https://openalex.org/W3029205506","https://openalex.org/W3092066298","https://openalex.org/W3092662090","https://openalex.org/W3093736937","https://openalex.org/W3094380520","https://openalex.org/W3094741512","https://openalex.org/W3100328669","https://openalex.org/W3125645198","https://openalex.org/W3140930832","https://openalex.org/W3153326528","https://openalex.org/W3190695173","https://openalex.org/W3195066380","https://openalex.org/W3217222343","https://openalex.org/W4256444741"],"related_works":["https://openalex.org/W2566658409","https://openalex.org/W3119324922","https://openalex.org/W2352686120","https://openalex.org/W2372594123","https://openalex.org/W2358310581","https://openalex.org/W2026132847","https://openalex.org/W4385695127","https://openalex.org/W2137810919","https://openalex.org/W2964752624","https://openalex.org/W4255854114"],"abstract_inverted_index":{"Web":[0,21,135],"measurement":[1,86,136,145],"studies":[2,87],"can":[3],"shed":[4],"light":[5],"on":[6,138],"not":[7,52],"yet":[8],"fully":[9],"understood":[10],"phenomena":[11],"and":[12,28,57,88,112,118,170,175],"thus":[13],"are":[14,51,66,121],"essential":[15,115],"for":[16,43,84],"analyzing":[17],"how":[18],"the":[19,55,60,103,109,126,149,152,162,167],"modern":[20],"works.":[22],"This":[23],"often":[24,122],"requires":[25],"building":[26],"new":[27],"adjusting":[29],"existing":[30],"crawling":[31],"setups,":[32],"which":[33],"has":[34],"led":[35],"to":[36,68,80,93,102,116,147],"a":[37,133],"wide":[38],"variety":[39],"of":[40,59,128,151],"analysis":[41],"tools":[42],"different":[44,144],"(but":[45],"related)":[46],"aspects.":[47],"If":[48],"these":[49,100],"efforts":[50],"sufficiently":[53],"documented,":[54],"reproducibility":[56],"replicability":[58],"measurements":[61],"may":[62],"suffer\u2014two":[63],"properties":[64],"that":[65,91,108,158],"crucial":[67],"sustainable":[69],"research.":[70],"In":[71],"this":[72,129],"paper,":[73],"we":[74,106],"survey":[75],"117":[76],"recent":[77],"research":[78],"papers":[79],"derive":[81],"best":[82],"practices":[83],"Web-based":[85],"specify":[89],"criteria":[90,101],"need":[92],"be":[94,172],"met":[95],"in":[96,161],"practice.":[97],"When":[98],"applying":[99],"surveyed":[104],"papers,":[105],"find":[107],"experimental":[110,163],"setup":[111,164],"other":[113],"aspects":[114],"reproducing":[117],"replicating":[119],"results":[120,169],"missing.":[123],"We":[124],"underline":[125],"criticality":[127],"finding":[130],"by":[131],"performing":[132],"large-scale":[134],"study":[137],"4.5":[139],"million":[140],"pages":[141],"with":[142],"24":[143],"setups":[146],"demonstrate":[148],"influence":[150],"individual":[153],"criteria.":[154],"Our":[155],"experiments":[156],"show":[157],"slight":[159],"differences":[160],"directly":[165],"affect":[166],"overall":[168],"must":[171],"documented":[173],"accurately":[174],"carefully.":[176]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":12},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":3}],"updated_date":"2026-08-11T07:18:39.950985","created_date":"2025-10-10T00:00:00"}