{"id":"https://openalex.org/W7138192624","doi":"https://doi.org/10.48550/arxiv.2603.15416","title":"Estimating Absolute Web Crawl Coverage From Longitudinal Set Intersections","display_name":"Estimating Absolute Web Crawl Coverage From Longitudinal Set Intersections","publication_year":2026,"publication_date":"2026-03-16","ids":{"openalex":"https://openalex.org/W7138192624","doi":"https://doi.org/10.48550/arxiv.2603.15416"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.15416","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.15416","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.15416","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5025133040","display_name":"Michael Par\u00eds","orcid":"https://orcid.org/0000-0003-2077-6984"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paris, Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003451290","display_name":"Grigori Paris","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paris, Grigori","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139731324","display_name":"Fabian Baumann","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Baumann, Fabian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.965399980545044,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.965399980545044,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13976","display_name":"Web visibility and informetrics","score":0.01730000041425228,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.0017999999690800905,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/completeness","display_name":"Completeness (order theory)","score":0.5412999987602234},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5317000150680542},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.4869999885559082},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.46779999136924744},{"id":"https://openalex.org/keywords/longitudinal-data","display_name":"Longitudinal data","score":0.4007999897003174},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.39719998836517334}],"concepts":[{"id":"https://openalex.org/C17231256","wikidata":"https://www.wikidata.org/wiki/Q5156540","display_name":"Completeness (order theory)","level":2,"score":0.5412999987602234},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5317000150680542},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5113000273704529},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5022000074386597},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.4869999885559082},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.46779999136924744},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.41429999470710754},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40310001373291016},{"id":"https://openalex.org/C3020672099","wikidata":"https://www.wikidata.org/wiki/Q857354","display_name":"Longitudinal data","level":2,"score":0.4007999897003174},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.39719998836517334},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34369999170303345},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3127000033855438},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.3043999969959259},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2992999851703644},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.28209999203681946},{"id":"https://openalex.org/C154775046","wikidata":"https://www.wikidata.org/wiki/Q188","display_name":"German","level":2,"score":0.27320000529289246},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.26269999146461487}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.15416","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.15416","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.15416","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.15416","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.46849897503852844},{"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10","score":0.420271098613739}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Web":[0],"archives":[1],"preserve":[2],"portions":[3],"of":[4,19,27,35,53,101,122,137,141],"the":[5,17,25,33,50,58,64,81,102,123,142,147],"web,":[6],"but":[7],"quantifying":[8],"their":[9],"completeness":[10],"remains":[11],"challenging.":[12],"Prior":[13],"approaches":[14],"have":[15],"estimated":[16,79],"coverage":[18,52,76,136],"a":[20,36,46,54,95,135],"crawl":[21,38,55,111,120,149],"by":[22,31,67,94],"either":[23],"comparing":[24,32],"outcomes":[26],"multiple":[28,68],"crawlers,":[29],"or":[30],"results":[34],"single":[37],"to":[39,48,117,164],"external":[40,159],"ground":[41,160],"truth":[42],"datasets.":[43],"We":[44],"propose":[45],"method":[47,153],"estimate":[49],"absolute":[51],"using":[56,113],"only":[57],"archive's":[59],"own":[60],"longitudinal":[61,110,166],"data,":[62],"i.e.,":[63],"data":[65,112],"collected":[66],"subsequent":[69,86],"crawls.":[70],"Our":[71,152],"key":[72],"insight":[73],"is":[74,154],"that":[75],"can":[77,105],"be":[78,107],"from":[80,109],"empirical":[82],"URL":[83,144],"overlaps":[84],"between":[85,131],"crawls,":[87],"which":[88],"are":[89],"in":[90],"turn":[91],"well":[92],"described":[93],"simple":[96],"urn":[97,103],"process.":[98],"The":[99],"parameters":[100],"model":[104],"then":[106],"inferred":[108],"linear":[114],"regression.":[115],"Applied":[116],"our":[118],"focused":[119,167],"configuration":[121,150],"German":[124],"Academic":[125],"Web,":[126],"with":[127],"15":[128],"semi-annual":[129],"crawls":[130],"2013-2021,":[132],"we":[133],"find":[134],"approximately":[138],"46":[139],"percent":[140],"crawlable":[143],"space":[145],"for":[146],"stable":[148],"regime.":[151],"extremely":[155],"simple,":[156],"requires":[157],"no":[158],"truth,":[161],"and":[162],"generalizes":[163],"any":[165],"crawl.":[168]},"counts_by_year":[],"updated_date":"2026-08-12T21:12:35.861297","created_date":"2026-02-03T00:00:00"}