{"id":"https://openalex.org/W7166841539","doi":"https://doi.org/10.18653/v1/2026.acl-long.11","title":"Benchmarking LLM\u2019s Capability in Reasoning over Conflicting Web References","display_name":"Benchmarking LLM\u2019s Capability in Reasoning over Conflicting Web References","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166841539","doi":"https://doi.org/10.18653/v1/2026.acl-long.11"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.acl-long.11","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.11","pdf_url":"https://aclanthology.org/2026.acl-long.11.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.acl-long.11.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139783499","display_name":"Yizhen Yuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yizhen Yuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139800649","display_name":"Rui Kong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rui Kong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106404994","display_name":"Dongze Li","orcid":"https://orcid.org/0009-0009-4567-1506"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dongze Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139758926","display_name":"Yuanchun Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuanchun Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139745615","display_name":"Yunxin Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yunxin Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.83141824,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"303","last_page":"322"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.3122999966144562,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.3122999966144562,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11010","display_name":"Logic, Reasoning, and Knowledge","score":0.09449999779462814,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.07680000364780426,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.5171999931335449},{"id":"https://openalex.org/keywords/knowledge-representation-and-reasoning","display_name":"Knowledge representation and reasoning","score":0.2840999960899353},{"id":"https://openalex.org/keywords/semantic-web","display_name":"Semantic Web","score":0.26440000534057617},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.25859999656677246},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.25679999589920044}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6869000196456909},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.5171999931335449},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3483000099658966},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.30379998683929443},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2856000065803528},{"id":"https://openalex.org/C161301231","wikidata":"https://www.wikidata.org/wiki/Q3478658","display_name":"Knowledge representation and reasoning","level":2,"score":0.2840999960899353},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2809999883174896},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.27129998803138733},{"id":"https://openalex.org/C2129575","wikidata":"https://www.wikidata.org/wiki/Q54837","display_name":"Semantic Web","level":2,"score":0.26440000534057617},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.25859999656677246},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.25679999589920044},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2540999948978424},{"id":"https://openalex.org/C20162079","wikidata":"https://www.wikidata.org/wiki/Q1151406","display_name":"Case-based reasoning","level":2,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.acl-long.11","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.11","pdf_url":"https://aclanthology.org/2026.acl-long.11.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.acl-long.11","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-long.11","pdf_url":"https://aclanthology.org/2026.acl-long.11.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4968693648","display_name":null,"funder_award_id":"62272261","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6396942078","display_name":null,"funder_award_id":"20242001120","funder_id":"https://openalex.org/F4320322392","funder_display_name":"Tsinghua University"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322392","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166841539.pdf","grobid_xml":"https://content.openalex.org/works/W7166841539.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,180],"(LLMs)":[3],"integrated":[4],"with":[5,23,89,93,134,178],"retrieval-augmented":[6],"generation":[7,82],"(RAG)":[8],"have":[9],"become":[10],"a":[11,112,169],"dominant":[12],"framework":[13],"for":[14,114,193],"building":[15],"intelligent":[16],"assistants.In":[17],"real-world":[18,120,130,218],"applications":[19],"such":[20,181],"as":[21,182],"ChatGPT":[22],"web":[24,220],"search,":[25],"the":[26,60,125,149,191,201,208],"retrieved":[27,57,123,139],"document":[28],"often":[29],"comes":[30],"from":[31,124,141],"diverse,":[32],"potentially":[33],"unreliable":[34],"sources":[35],"and":[36,69,80,102,164,174,184],"may":[37],"contain":[38],"inconsistent":[39],"claims.Unlike":[40],"traditional":[41],"search":[42],"engines":[43],"that":[44,83],"rely":[45],"on":[46,87,217],"users":[47],"to":[48,65,100,167,172,213],"manually":[49],"compare":[50],"information,":[51],"LLM-based":[52],"systems":[53],"typically":[54],"feed":[55],"all":[56],"content":[58],"into":[59],"model's":[61,170],"context,":[62,91],"requiring":[63],"LLMs":[64],"autonomously":[66],"identify,":[67],"differentiate,":[68],"reason":[70,103,165],"over":[71,104,119],"conflicting":[72,121],"viewpoints.Unlike":[73],"mainstream":[74],"LLM":[75],"evaluation":[76,158],"tasks":[77],"like":[78],"math":[79],"code":[81],"are":[84],"primarily":[85],"focused":[86],"reasoning":[88,117,216],"factual":[90],"question-answering":[92],"multi-source":[94],"references":[95],"requires":[96],"fundamentally":[97],"different":[98],"capabilities":[99],"identify":[101],"knowledge":[105],"contradictions.In":[106],"this":[107],"paper,":[108],"we":[109],"introduce":[110],"CONFRAG,":[111],"benchmark":[113,210],"evaluating":[115],"LLMs'":[116],"capability":[118],"documents":[122],"web.It":[126],"consists":[127],"of":[128,137,146,148,203],"1,814":[129],"questions,":[131],"each":[132],"paired":[133],"an":[135],"average":[136],"9.58":[138],"paragraphs":[140],"heterogeneous":[142],"online":[143],"sources.A":[144],"total":[145],"57.2%":[147],"questions":[150],"exhibit":[151],"explicit":[152],"contradictions.We":[153],"further":[154],"propose":[155],"three":[156],"structured":[157],"tasks,":[159],"answer":[160,162],"clustering,":[161],"coverage,":[163,166],"quantify":[168],"ability":[171],"organize":[173],"explain":[175],"contradictory":[176],"content.Experiments":[177],"state-of-the-art":[179],"GPT-4.1":[183],"Claude-3-7-Sonnet":[185],"reveal":[186],"substantial":[187],"performance":[188],"gaps,":[189],"highlighting":[190],"need":[192],"more":[194],"targeted":[195],"research":[196],"in":[197],"contradiction-aware":[198,215],"question":[199],"answering.To":[200],"best":[202],"our":[204],"knowledge,":[205],"CONFRAG":[206],"is":[207],"first":[209],"specifically":[211],"designed":[212],"evaluate":[214],"long":[219],"documents.":[221]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-07-02T00:00:00"}