{"id":"https://openalex.org/W7166807136","doi":"https://doi.org/10.18653/v1/2026.acl-industry.132","title":"Entity Exchange in the Wild: A Diagnostic Study of LLM Based Real-World Conversational Entity Extraction","display_name":"Entity Exchange in the Wild: A Diagnostic Study of LLM Based Real-World Conversational Entity Extraction","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166807136","doi":"https://doi.org/10.18653/v1/2026.acl-industry.132"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2026.acl-industry.132","doi":"https://doi.org/10.18653/v1/2026.acl-industry.132","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-industry.132","pdf_url":"https://aclanthology.org/2026.acl-industry.132.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 6: Industry Track)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2026.acl-industry.132.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103967035","display_name":"Soumya Jain","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Soumya Jain","raw_affiliation_strings":["Observe.AI Bangalore , India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Observe.AI Bangalore , India","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5139837360","display_name":"Ayush Kumar","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ayush Kumar","raw_affiliation_strings":["Observe.AI Bangalore , India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Observe.AI Bangalore , India","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.43490111,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1933","last_page":"1955"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.7324000000953674,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.7324000000953674,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.23360000550746918,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.016100000590085983,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.753000020980835},{"id":"https://openalex.org/keywords/spoken-language-understanding","display_name":"spoken language understanding","score":0.3540000021457672},{"id":"https://openalex.org/keywords/llm-benchmarking","display_name":"LLM benchmarking","score":0.5830000042915344},{"id":"https://openalex.org/keywords/llm-as-a-judge-evaluation","display_name":"LLM-as-judge evaluation","score":0.3569999933242798}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6470999717712402},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5703999996185303},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5573999881744385},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.3133000135421753},{"id":"https://openalex.org/C195807954","wikidata":"https://www.wikidata.org/wiki/Q1662562","display_name":"Information extraction","level":2,"score":0.3021000027656555},{"id":"https://openalex.org/C4725764","wikidata":"https://www.wikidata.org/wiki/Q844704","display_name":"Extraction (chemistry)","level":2,"score":0.2702000141143799},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.26660001277923584},{"id":"https://openalex.org/C96711827","wikidata":"https://www.wikidata.org/wiki/Q17012245","display_name":"Entity linking","level":3,"score":0.2632000148296356},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.260699987411499},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.2506999969482422}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.acl-industry.132","doi":"https://doi.org/10.18653/v1/2026.acl-industry.132","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-industry.132","pdf_url":"https://aclanthology.org/2026.acl-industry.132.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 6: Industry Track)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.acl-industry.132","doi":"https://doi.org/10.18653/v1/2026.acl-industry.132","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-industry.132","pdf_url":"https://aclanthology.org/2026.acl-industry.132.pdf","source":{"id":"https://openalex.org/S4393915740","display_name":"Proceedings of the conference - Association for Computational Linguistics. Meeting","issn_l":"0736-587X","issn":["0736-587X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["norway-1"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 6: Industry Track)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166807136.pdf","grobid_xml":"https://content.openalex.org/works/W7166807136.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7168276025","https://openalex.org/W4389519581","https://openalex.org/W2566356701","https://openalex.org/W7140077301","https://openalex.org/W7213371868","https://openalex.org/W7163203416","https://openalex.org/W4310561787","https://openalex.org/W2110462621","https://openalex.org/W4416037168","https://openalex.org/W2945614092"],"abstract_inverted_index":{"Entity":[0,207,242],"extraction":[1,14,47,162,185],"from":[2,278],"spoken":[3,423],"customer-agent":[4],"conversations":[5,96],"is":[6,83,130,243,254,348],"increasingly":[7],"driving":[8],"automation":[9],"in":[10,13,85,186,212],"contact":[11],"centers.Errors":[12],"can":[15],"trigger":[16,301,441],"incorrect":[17],"system":[18,416],"actions":[19],"such":[20],"as":[21],"database":[22],"updates,":[23],"verification":[24],"failures,":[25],"or":[26,267,329],"workflow":[27],"execution.While":[28],"prior":[29,248,395],"work":[30,177],"has":[31,39],"examined":[32],"transcription":[33],"noise":[34],"and":[35,74,105,135,151,189],"cross-turn":[36],"reasoning,":[37],"it":[38,309],"not":[40,307],"systematically":[41],"analyzed":[42],"how":[43,199,291,308],"entityexchange":[44],"phenomena":[45,51],"shape":[46],"performance.We":[48],"model":[49,137],"these":[50],"along":[52],"three":[53,165],"orthogonal":[54],"axes:":[55],"Initiation":[56,194],"(how":[57,63,76],"an":[58,69,81,200,215],"entity":[59,70,82,99,128,184,201,293,305,320,333,360,372,389,445],"becomes":[60,262],"relevant),":[61],"Evolution":[62],"commitment":[64],"to":[65,111,214,408],"the":[66,77,114,119,127,133,136,204,210,292,296,300,304,319,356,359,388,420,432,439,444],"value":[67,79,129,421,433],"of":[68,80,158,170],"develops":[71],"across":[72,101,163,173],"turns),":[73],"Articulation":[75],"final":[78],"expressed":[84],"surface":[86],"form).We":[87],"evaluate":[88],"16":[89],"large":[90],"language":[91],"models":[92],"on":[93,118,290,299,438],"6,387":[94],"realworld":[95],"spanning":[97],"12":[98],"types":[100],"numeric,":[102],"alphanumeric,":[103],"temporal,":[104],"free-text":[106],"categories.Performance":[107],"varies":[108],"by":[109],"up":[110],"50-60%":[112],"within":[113],"same":[115],"LLM":[116],"depending":[117],"entity-exchange":[120],"phenomena.The":[121],"most":[122],"severe":[123],"failures":[124],"occur":[125],"when":[126,146],"revised":[131],"during":[132],"interaction":[134],"must":[138],"contrast":[139],"intermediate":[140],"values":[141],"with":[142,375],"a":[143,155,179,247,255,327],"committed":[144],"value.Even":[145],"no":[147],"revision":[148],"occurs,":[149],"digit-bydigit":[150],"encoded":[152],"expressions":[153],"remain":[154],"consistent":[156],"source":[157],"error.Error-Aware":[159],"prompting":[160],"improves":[161],"all":[164,174],"axes,":[166],"yielding":[167],"average":[168],"gains":[169],"upto":[171],"6.4%":[172],"LLMs.Together,":[175],"this":[176,332],"provides":[178],"structured":[180],"basis":[181],"for":[182,331],"benchmarking":[183],"real-world":[187],"deployments":[188],"isolating":[190],"systematic":[191],"failure":[192],"modes.":[193],"Taxonomy":[195],"Prompt":[196],"INITIATION":[197],"TAXONOMYModels":[198],"first":[202,208,294,321],"enters":[203,209,295],"conversation.I0.Elicited":[205],"Entry":[206,241],"conversation":[211],"response":[213],"explicit":[216],"request/prompt.\u2022":[217,249],"Example:":[218,229,250,270],"\"May":[219,340],"I":[220,341],"have":[221],"your":[222,233,279,412],"order":[223,273],"number?\"\u2192":[224,235],"\"o":[225],"nine":[226,275],"five":[227,237,257,276],"four\"\u2022":[228],"\"Do":[230],"you":[231,344,351],"want":[232],"case":[234],"\"three":[236],"seven":[238],"six\"":[239],"I1.Volunteered":[240],"introduced":[244],"proactively":[245],"without":[246],"\"My":[251,376],"account":[252],"ID":[253],"zero":[256],"nine..\"":[258],"I2.":[259],"Context-Triggered":[260],"EntryEntity":[261],"relevant":[263],"implicitly":[264],"through":[265],"conversational":[266,440],"external":[268,397],"context.\u2022":[269],"\"I":[271,379,405],"see":[272],"o":[274],"four":[277],"previous":[280,413],"case\"":[281],"ASSIGNMENT":[282],"RULE:\u2022":[283],"Choose":[284],"EXACTLY":[285],"ONE":[286],"initiation":[287],"label":[288],"based":[289],"conversation.\u2022":[297],"Focus":[298,436],"that":[302,442],"made":[303],"relevant,":[306],"was":[310,326,390,422],"articulated.RULE":[311],"OF":[312],"THUMB:":[313],"Ask:":[314],"\"What":[315,347],"happened":[316],"IMMEDIATELY":[317],"BEFORE":[318],"appeared?\"\u2022":[322],"\u2192":[323,334,354,364,386,399],"If":[324,355,387],"there":[325],"QUESTION":[328],"REQUEST":[330],"I0":[335],"(Elicited":[336],"Entry)":[337,367,402],"Key":[338,368,403],"signals:":[339,369,404],"have..?\",":[342],"\"Can":[343],"give":[345,383],"me..?\",":[346],"your..?\",":[349],"\"Could":[350],"provide..?\"":[352],"\u2022":[353,385],"speaker":[357],"OFFERED":[358],"WITHOUT":[361],"being":[362],"asked":[363],"I1":[365],"(Volunteered":[366],"Speaker":[370],"introduces":[371],"mid-conversation,":[373],"often":[374],"X":[377],"is...\",":[378],"have...\",":[380],"\"Let":[381],"me":[382],"you...\"":[384],"PULLED":[391],"FROM":[392],"CONTEXT":[393],"(system,":[394],"conversation,":[396],"source)":[398],"I2":[400],"(Context-Triggered":[401],"see...\",":[406],"\"According":[407],"our":[409],"records...\",":[410],"\"From":[411],"case...\",":[414],"\"The":[415],"shows...\"IMPORTANT:\u2022":[417],"Ignore":[418,430],"HOW":[419],"(spelled":[424],"out,":[425],"fragmented,":[426],"etc.)":[427],"-that's":[428],"Articulation.\u2022":[429],"WHETHER":[431],"changed":[434],"later.\u2022":[435],"ONLY":[437],"brought":[443],"into":[446],"discussion.":[447]},"counts_by_year":[],"updated_date":"2026-10-29T09:56:32.521200","created_date":"2026-07-02T00:00:00"}