{"id":"https://openalex.org/W7219650090","doi":"https://doi.org/10.5281/zenodo.23142233","title":"WebRuleBench: An open-source environment for benchmarking LLM-generated web extraction rules against human-annotated ground truth","display_name":"WebRuleBench: An open-source environment for benchmarking LLM-generated web extraction rules against human-annotated ground truth","publication_year":2026,"publication_date":"2026-10-04","ids":{"openalex":"https://openalex.org/W7219650090","doi":"https://doi.org/10.5281/zenodo.23142233"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.23142233","doi":"https://doi.org/10.5281/zenodo.23142233","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23142233","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},"volume":null,"type":"software","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.23142233","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006050193","display_name":"Burak Sevinc","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I125454184","display_name":"Tekirda\u011f Nam\u0131k Kemal University","ror":"https://ror.org/01a0mk874","country_code":"TR","type":"education","lineage":["https://openalex.org/I125454184"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Sevin\u00e7, Burak","raw_affiliation_strings":["Tekirda\u011f Nam\u0131k Kemal University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tekirda\u011f Nam\u0131k Kemal University","institution_ids":["https://openalex.org/I125454184"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078816512","display_name":"Erdin\u00e7 Uzun","orcid":"https://orcid.org/0000-0003-4351-2244","observed_orcids":["https://orcid.org/0000-0003-4351-2244"]},"institutions":[{"id":"https://openalex.org/I125454184","display_name":"Tekirda\u011f Nam\u0131k Kemal University","ror":"https://ror.org/01a0mk874","country_code":"TR","type":"education","lineage":["https://openalex.org/I125454184"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Uzun, Erdin\u00e7","raw_affiliation_strings":["Tekirda\u011f Nam\u0131k Kemal University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tekirda\u011f Nam\u0131k Kemal University","institution_ids":["https://openalex.org/I125454184"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I125454184"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.8560000061988831},{"id":"https://openalex.org/keywords/open-source-software","display_name":"open source software","score":0.5619999766349792},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7039999961853027},{"id":"https://openalex.org/keywords/rule-evaluation","display_name":"rule evaluation","score":0.32499998807907104}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7142999768257141},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7024999856948853},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.6676999926567078},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5720999836921692},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.46779999136924744},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4489000141620636},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.44690001010894775},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4284999966621399},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.42489999532699585},{"id":"https://openalex.org/C2987528665","wikidata":"https://www.wikidata.org/wiki/Q854845","display_name":"Earth observation satellite","level":3,"score":0.3790000081062317},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3319000005722046},{"id":"https://openalex.org/C195807954","wikidata":"https://www.wikidata.org/wiki/Q1662562","display_name":"Information extraction","level":2,"score":0.31470000743865967},{"id":"https://openalex.org/C21959979","wikidata":"https://www.wikidata.org/wiki/Q36774","display_name":"Web page","level":2,"score":0.31439998745918274},{"id":"https://openalex.org/C118643609","wikidata":"https://www.wikidata.org/wiki/Q189210","display_name":"Web application","level":2,"score":0.31299999356269836},{"id":"https://openalex.org/C72634772","wikidata":"https://www.wikidata.org/wiki/Q386824","display_name":"Data integration","level":2,"score":0.31049999594688416},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.30160000920295715},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.295199990272522},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2946999967098236},{"id":"https://openalex.org/C2983685735","wikidata":"https://www.wikidata.org/wiki/Q5227355","display_name":"Data source","level":2,"score":0.2921999990940094},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.27399998903274536},{"id":"https://openalex.org/C2777877512","wikidata":"https://www.wikidata.org/wiki/Q1116097","display_name":"Common ground","level":2,"score":0.262800008058548},{"id":"https://openalex.org/C4725764","wikidata":"https://www.wikidata.org/wiki/Q844704","display_name":"Extraction (chemistry)","level":2,"score":0.2587999999523163},{"id":"https://openalex.org/C2777466982","wikidata":"https://www.wikidata.org/wiki/Q5227287","display_name":"Data extraction","level":3,"score":0.2563000023365021}],"mesh":[],"locations_count":6,"locations":[{"id":"doi:10.5281/zenodo.23142233","doi":"https://doi.org/10.5281/zenodo.23142233","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23142233","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.23142234","doi":"https://doi.org/10.5281/zenodo.23142234","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23142234","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.23145117","doi":"https://doi.org/10.5281/zenodo.23145117","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23145117","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.23169964","doi":"https://doi.org/10.5281/zenodo.23169964","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23169964","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.23180562","doi":"https://doi.org/10.5281/zenodo.23180562","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23180562","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},{"id":"doi:10.5281/zenodo.23192130","doi":"https://doi.org/10.5281/zenodo.23192130","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23192130","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"}],"best_oa_location":{"id":"doi:10.5281/zenodo.23142233","doi":"https://doi.org/10.5281/zenodo.23142233","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23142233","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Software"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7219766513","https://openalex.org/W7166888181","https://openalex.org/W7169866231","https://openalex.org/W7166885419","https://openalex.org/W4415154343","https://openalex.org/W7162809613","https://openalex.org/W7090275347","https://openalex.org/W7154951376","https://openalex.org/W7213237198","https://openalex.org/W7213508891"],"abstract_inverted_index":{"First":[0],"public":[1],"release":[2],"(SoftwareX":[3],"submission).":[4],"Web":[5],"application":[6],"with":[7,50,56,117,133,141,155],"six":[8],"sections":[9],"\u2014":[10,24,162],"data":[11],"collection,":[12],"templates":[13],"&":[14,22],"prompts,":[15],"annotation,":[16],"ground-truth":[17,54,92],"approval,":[18],"LLM":[19,28,81,84,104,160],"evaluation,":[20],"reports":[21],"export":[23,132],"and":[25,30,33,45,65,112,124,145,158],"settings":[26],"for":[27],"backends":[29],"users;":[31],"English":[32],"Turkish":[34],"interface.":[35],"Multi-annotator":[36],"rule":[37],"collection":[38],"in":[39,76],"CSS,":[40],"XPath":[41],"(suggested":[42],"from":[43,48],"CSS)":[44],"regex":[46],"(generated":[47],"CSS":[49],"an":[51,146],"extended":[52],"REGEXN);":[53],"approval":[55],"rule-level":[57],"agreement":[58],"(normalized":[59],"rules,":[60],"Fleiss'":[61],"\u03ba":[62],"over":[63],"layouts)":[64],"content-level":[66],"scores":[67],"(ROUGE":[68],"/":[69,71,119,121,128,130],"Jaccard":[70],"exact":[72],"match;":[73],"Krippendorff's":[74],"\u03b1":[75],"the":[77],"study":[78,166],"script).":[79],"Layout-based":[80],"experiments:":[82],"one":[83],"call":[85],"per":[86],"layout,":[87],"rules":[88],"scored":[89],"on":[90],"every":[91],"page,":[93],"sample-page":[94],"vs.":[95],"other-page":[96],"scores,":[97],"comparative":[98],"experiments,":[99],"cross-check,":[100],"quick":[101],"test,":[102],"resume.":[103],"backend":[105],"registry:":[106],"Ollama,":[107],"Anthropic,":[108],"Gemini,":[109],"OpenAI-compatible":[110],"endpoints":[111],"a":[113,134],"replay":[114],"backend.":[115],"Reports":[116],"CSV":[118,129],"LaTeX":[120],"Markdown":[122],"tables":[123],"PNG":[125],"charts;":[126],"JSONL":[127],"HuggingFace":[131],"dataset":[135],"card;":[136],"reproducibility":[137],"manifest.":[138],"Token-based":[139],"users":[140],"salted":[142],"hashes,":[143],"roles":[144],"audit":[147],"log.":[148],"Reviewer":[149],"demo":[150],"(webrulebench":[151],"demo,":[152],"./run_demo.sh,":[153],"Dockerfile)":[154],"synthetic":[156],"sites":[157],"recorded":[159],"responses":[161],"see":[163],"demo/README.md.":[164],"Annotation":[165],"scripts":[167],"(paper/annotation_study/).":[168]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-10-05T00:00:00"}