{"id":"https://openalex.org/W4416549320","doi":"https://doi.org/10.1145/3719027.3765063","title":"The Odyssey of robots.txt Governance: Measuring Convention Implications of Web Bots in Large Language Model Services","display_name":"The Odyssey of robots.txt Governance: Measuring Convention Implications of Web Bots in Large Language Model Services","publication_year":2025,"publication_date":"2025-11-19","ids":{"openalex":"https://openalex.org/W4416549320","doi":"https://doi.org/10.1145/3719027.3765063"},"language":"en","primary_location":{"id":"doi:10.1145/3719027.3765063","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3719027.3765063","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 ACM SIGSAC Conference on Computer and Communications Security","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3719027.3765063","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100763243","display_name":"Jian Cui","orcid":"https://orcid.org/0000-0002-5212-0501"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jian Cui","raw_affiliation_strings":["University of Illinois Urbana-Champaign, Champaign, USA"],"raw_orcid":"https://orcid.org/0000-0002-5212-0501","affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign, Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002143912","display_name":"Mingming Zha","orcid":"https://orcid.org/0000-0002-7827-9369"},"institutions":[{"id":"https://openalex.org/I4210119109","display_name":"Indiana University Bloomington","ror":"https://ror.org/02k40bc56","country_code":"US","type":"education","lineage":["https://openalex.org/I4210119109","https://openalex.org/I592451"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mingming Zha","raw_affiliation_strings":["Indiana University Bloomington, Bloomington, USA"],"raw_orcid":"https://orcid.org/0000-0002-7827-9369","affiliations":[{"raw_affiliation_string":"Indiana University Bloomington, Bloomington, USA","institution_ids":["https://openalex.org/I4210119109"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066976667","display_name":"XiaoFeng Wang","orcid":"https://orcid.org/0000-0002-0607-4946"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"XiaoFeng Wang","raw_affiliation_strings":["Nanyang Technological University, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-0607-4946","affiliations":[{"raw_affiliation_string":"Nanyang Technological University, Singapore, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5084889167","display_name":"Xiaojing Liao","orcid":"https://orcid.org/0000-0001-7555-1673"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaojing Liao","raw_affiliation_strings":["University of Illinois Urbana-Champaign, Champaign, USA"],"raw_orcid":"https://orcid.org/0000-0001-7555-1673","affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign, Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"21","last_page":"35"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.16279999911785126,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.16279999911785126,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.09000000357627869,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.061500001698732376,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/web-standards","display_name":"Web standards","score":0.6572999954223633},{"id":"https://openalex.org/keywords/web-development","display_name":"Web development","score":0.6316999793052673},{"id":"https://openalex.org/keywords/web-application-security","display_name":"Web application security","score":0.5275999903678894},{"id":"https://openalex.org/keywords/web-service","display_name":"Web service","score":0.5105999708175659},{"id":"https://openalex.org/keywords/web-content","display_name":"Web content","score":0.48590001463890076},{"id":"https://openalex.org/keywords/transparency","display_name":"Transparency (behavior)","score":0.4311000108718872},{"id":"https://openalex.org/keywords/web-accessibility-initiative","display_name":"Web Accessibility Initiative","score":0.4284999966621399},{"id":"https://openalex.org/keywords/web-design","display_name":"Web design","score":0.36820000410079956}],"concepts":[{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.6868000030517578},{"id":"https://openalex.org/C182321512","wikidata":"https://www.wikidata.org/wiki/Q1153289","display_name":"Web standards","level":3,"score":0.6572999954223633},{"id":"https://openalex.org/C79373723","wikidata":"https://www.wikidata.org/wiki/Q386275","display_name":"Web development","level":3,"score":0.6316999793052673},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5536999702453613},{"id":"https://openalex.org/C59241245","wikidata":"https://www.wikidata.org/wiki/Q4781497","display_name":"Web application security","level":4,"score":0.5275999903678894},{"id":"https://openalex.org/C35578498","wikidata":"https://www.wikidata.org/wiki/Q193424","display_name":"Web service","level":2,"score":0.5105999708175659},{"id":"https://openalex.org/C2776324614","wikidata":"https://www.wikidata.org/wiki/Q3948731","display_name":"Web content","level":3,"score":0.48590001463890076},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.4311000108718872},{"id":"https://openalex.org/C197973564","wikidata":"https://www.wikidata.org/wiki/Q636020","display_name":"Web Accessibility Initiative","level":5,"score":0.4284999966621399},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.4115000069141388},{"id":"https://openalex.org/C521306242","wikidata":"https://www.wikidata.org/wiki/Q190637","display_name":"Web design","level":3,"score":0.36820000410079956},{"id":"https://openalex.org/C136699151","wikidata":"https://www.wikidata.org/wiki/Q131164","display_name":"Web 2.0","level":3,"score":0.36559998989105225},{"id":"https://openalex.org/C190515685","wikidata":"https://www.wikidata.org/wiki/Q749179","display_name":"WS-Policy","level":5,"score":0.3589000105857849},{"id":"https://openalex.org/C130436687","wikidata":"https://www.wikidata.org/wiki/Q7978591","display_name":"Web modeling","level":3,"score":0.3456999957561493},{"id":"https://openalex.org/C516187249","wikidata":"https://www.wikidata.org/wiki/Q10719477","display_name":"Web analytics","level":5,"score":0.33649998903274536},{"id":"https://openalex.org/C162446236","wikidata":"https://www.wikidata.org/wiki/Q653137","display_name":"Content analysis","level":2,"score":0.32339999079704285},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.31439998745918274},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.3116999864578247},{"id":"https://openalex.org/C97200028","wikidata":"https://www.wikidata.org/wiki/Q1196135","display_name":"Web engineering","level":5,"score":0.31119999289512634},{"id":"https://openalex.org/C33039251","wikidata":"https://www.wikidata.org/wiki/Q173373","display_name":"Content management","level":2,"score":0.3073999881744385},{"id":"https://openalex.org/C39389867","wikidata":"https://www.wikidata.org/wiki/Q380767","display_name":"Corporate governance","level":2,"score":0.29910001158714294},{"id":"https://openalex.org/C162005631","wikidata":"https://www.wikidata.org/wiki/Q54837","display_name":"Data Web","level":3,"score":0.2800000011920929},{"id":"https://openalex.org/C541491724","wikidata":"https://www.wikidata.org/wiki/Q7094111","display_name":"Internet presence management","level":3,"score":0.2775000035762787}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3719027.3765063","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3719027.3765063","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 ACM SIGSAC Conference on Computer and Communications Security","raw_type":"proceedings-article"},{"id":"pmh:oai:dr.ntu.edu.sg:10356/204816","is_oa":false,"landing_page_url":"https://hdl.handle.net/10356/204816","pdf_url":null,"source":{"id":"https://openalex.org/S4306402609","display_name":"DR-NTU (Nanyang Technological University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I172675005","host_organization_name":"Nanyang Technological University","host_organization_lineage":["https://openalex.org/I172675005"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"Conference Paper"}],"best_oa_location":{"id":"doi:10.1145/3719027.3765063","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3719027.3765063","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 ACM SIGSAC Conference on Computer and Communications Security","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W1994863186","https://openalex.org/W2102475166","https://openalex.org/W2110989128","https://openalex.org/W2141105512","https://openalex.org/W2159185913","https://openalex.org/W2970641574","https://openalex.org/W2979826702","https://openalex.org/W2995463934","https://openalex.org/W3194548250","https://openalex.org/W4255268448","https://openalex.org/W4385679821","https://openalex.org/W4396757560","https://openalex.org/W4396758700","https://openalex.org/W4396843934"],"related_works":[],"abstract_inverted_index":{"Web":[0],"content":[1,21,34,39,47,72,100,183,205,213,222],"is":[2,84],"an":[3],"essential":[4],"element":[5],"for":[6,211],"large":[7],"language":[8],"model":[9],"(LLM)":[10],"services,":[11],"supporting":[12],"both":[13],"training":[14],"and":[15,61,74,97,140,163,187,192,207,221],"inference":[16],"processes.":[17],"To":[18,102],"manage":[19],"the":[20,51,68,138,144,157,160,164,199,202,209],"access":[22,40,73],"of":[23,53,70,87,111,159,166,174],"web":[24,33,46,71,95,99,147,182,204,218],"bots":[25,77,114],"from":[26,184],"LLM":[27,31,55,89,113,130,161],"service":[28],"vendors":[29],"(i.e.,":[30],"bots),":[32],"publishers":[35,96,148],"are":[36],"increasingly":[37],"incorporated":[38],"rules":[41,127],"into":[42,137],"robots.txt,":[43],"a":[44,108,122],"long-established":[45],"management":[48,214],"protocol.":[49],"However,":[50,82],"rise":[52],"proprietary":[54],"bots,":[56,90,131],"such":[57],"as":[58],"OpenAI's":[59],"ChatGPT-User":[60,189],"Google's":[62],"Google-Extended,":[63],"has":[64],"raised":[65],"concerns":[66],"about":[67],"transparency":[69],"whether":[75],"these":[76,88],"adherence":[78],"to":[79,156,216],"robots.txt":[80,117,126,153,175,191],"rules.":[81],"there":[83],"limited":[85],"understanding":[86],"concerning":[91],"their":[92],"impact":[93],"on":[94,115],"broader":[98],"governance.":[101],"fill":[103],"this":[104],"gap,":[105],"we":[106,170],"present":[107],"systematic":[109],"analysis":[110],"18":[112],"582,281":[116],"files.":[118],"Our":[119],"findings":[120],"reveal":[121],"significant":[123],"increase":[124],"in":[125,133,151,201],"associated":[128],"with":[129],"particularly":[132],"domains":[134],"that":[135],"fall":[136],"finance":[139],"news":[141],"category.":[142],"Despite":[143],"heightened":[145],"integration,":[146],"face":[149],"challenges":[150],"managing":[152],"configurations":[154],"due":[155],"complexity":[158],"ecosystem":[162],"involvement":[165],"third-party":[167],"brokers.":[168],"Furthermore,":[169],"identified":[171],"several":[172],"cases":[173],"violations,":[176],"including":[177],"instances":[178],"where":[179,188],"LLMs":[180],"memorized":[181],"restricted":[185,194],"domains,":[186],"ignored":[190],"accessed":[193],"content.":[195],"These":[196],"results":[197],"highlight":[198],"gaps":[200],"current":[203],"governance":[206],"underscore":[208],"need":[210],"enforceable":[212],"mechanisms":[215],"respect":[217],"publishers'":[219],"intentions":[220],"control.":[223]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-11-23T00:00:00"}