{"id":"https://openalex.org/W4401863263","doi":"https://doi.org/10.1145/3637528.3671796","title":"On Early Detection of Hallucinations in Factual Question Answering","display_name":"On Early Detection of Hallucinations in Factual Question Answering","publication_year":2024,"publication_date":"2024-08-24","ids":{"openalex":"https://openalex.org/W4401863263","doi":"https://doi.org/10.1145/3637528.3671796"},"language":"en","primary_location":{"id":"doi:10.1145/3637528.3671796","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3637528.3671796","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3637528.3671796","source":null,"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3637528.3671796","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5112036359","display_name":"Ben Snyder","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ben Snyder","raw_affiliation_strings":["Amazon Web Services, Santa Clara, CA, USA"],"affiliations":[{"raw_affiliation_string":"Amazon Web Services, Santa Clara, CA, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104219884","display_name":"Marius Moisescu","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Marius Moisescu","raw_affiliation_strings":["Amazon Web Services, Seattle, WA, USA"],"affiliations":[{"raw_affiliation_string":"Amazon Web Services, Seattle, WA, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102901191","display_name":"Muhammad Bilal Zafar","orcid":"https://orcid.org/0000-0001-8347-7813"},"institutions":[{"id":"https://openalex.org/I904495901","display_name":"Ruhr University Bochum","ror":"https://ror.org/04tsk2644","country_code":"DE","type":"education","lineage":["https://openalex.org/I904495901"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Muhammad Bilal Zafar","raw_affiliation_strings":["Ruhr-Universit\u00e4t Bochum & Research Center for Trustworthy Data Science and Security, University Alliance Ruhr, Bochum, Nordrhein-Westfalen, Germany"],"affiliations":[{"raw_affiliation_string":"Ruhr-Universit\u00e4t Bochum & Research Center for Trustworthy Data Science and Security, University Alliance Ruhr, Bochum, Nordrhein-Westfalen, Germany","institution_ids":["https://openalex.org/I904495901"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5112036359"],"corresponding_institution_ids":["https://openalex.org/I1311688040"],"apc_list":null,"apc_paid":null,"fwci":2.9026,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.91985813,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"2721","last_page":"2732"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.9868999719619751,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hallucinating","display_name":"Hallucinating","score":0.9808266758918762},{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.7140491008758545},{"id":"https://openalex.org/keywords/visual-hallucination","display_name":"Visual Hallucination","score":0.595770001411438},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.589427649974823},{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.5429614782333374},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.5258655548095703},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.47968125343322754},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.44036373496055603},{"id":"https://openalex.org/keywords/sarcasm","display_name":"Sarcasm","score":0.42580270767211914},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.39506059885025024},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3469087481498718},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3397061228752136},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.29379117488861084},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.20499315857887268},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.09626364707946777}],"concepts":[{"id":"https://openalex.org/C2911011789","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Hallucinating","level":2,"score":0.9808266758918762},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.7140491008758545},{"id":"https://openalex.org/C2908998935","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Visual Hallucination","level":2,"score":0.595770001411438},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.589427649974823},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.5429614782333374},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.5258655548095703},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47968125343322754},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.44036373496055603},{"id":"https://openalex.org/C2776207355","wikidata":"https://www.wikidata.org/wiki/Q191035","display_name":"Sarcasm","level":3,"score":0.42580270767211914},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.39506059885025024},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3469087481498718},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3397061228752136},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.29379117488861084},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.20499315857887268},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.09626364707946777},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C2779975665","wikidata":"https://www.wikidata.org/wiki/Q131361","display_name":"Irony","level":2,"score":0.0},{"id":"https://openalex.org/C118552586","wikidata":"https://www.wikidata.org/wiki/Q7867","display_name":"Psychiatry","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3637528.3671796","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3637528.3671796","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3637528.3671796","source":null,"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3637528.3671796","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3637528.3671796","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3637528.3671796","source":null,"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 30th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Gender equality","score":0.44999998807907104,"id":"https://metadata.un.org/sdg/5"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4401863263.pdf"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W2962772482","https://openalex.org/W2963339397","https://openalex.org/W2963847595","https://openalex.org/W2973136764","https://openalex.org/W3027879771","https://openalex.org/W3084095723","https://openalex.org/W3138819813","https://openalex.org/W3170432046","https://openalex.org/W4221163727","https://openalex.org/W4226278401","https://openalex.org/W4281657280","https://openalex.org/W4309674289","https://openalex.org/W4360980513","https://openalex.org/W4386566643","https://openalex.org/W4389519449","https://openalex.org/W4389520749","https://openalex.org/W4391156274","https://openalex.org/W4398757454","https://openalex.org/W4404534210","https://openalex.org/W6839328737"],"related_works":["https://openalex.org/W4285298148","https://openalex.org/W2169361442","https://openalex.org/W2496852906","https://openalex.org/W2043499099","https://openalex.org/W3168188161","https://openalex.org/W2613049420","https://openalex.org/W3168832442","https://openalex.org/W1998166138","https://openalex.org/W1965596860","https://openalex.org/W2589508179"],"abstract_inverted_index":{"While":[0],"large":[1],"language":[2],"models":[3],"(LLMs)":[4],"have":[5],"taken":[6],"great":[7],"strides":[8],"towards":[9,22],"helping":[10],"humans":[11],"with":[12,50],"a":[13,19,38,158],"plethora":[14],"of":[15,30,97,110],"tasks,":[16],"hallucinations":[17,98,141],"remain":[18],"major":[20],"impediment":[21],"gaining":[23],"user":[24],"trust.":[25],"The":[26],"fluency":[27],"and":[28,84,91,118,142],"coherence":[29],"model":[31,52,138],"generations":[32,53,139],"even":[33,166],"when":[34],"hallucinating":[35],"makes":[36],"detection":[37],"difficult":[39],"task.":[40],"In":[41],"this":[42,123],"work,":[43],"we":[44,64,125],"explore":[45],"if":[46],"the":[47,51,58,69,78,81,86,108,163],"artifacts":[48,112,132],"associated":[49],"can":[54,160],"provide":[55],"hints":[56],"that":[57,107,129,155],"generation":[59],"will":[60],"contain":[61],"hallucinations.":[62],"Specifically,":[63],"probe":[65],"LLMs":[66],"at":[67],"1)":[68],"inputs":[70],"via":[71,80,89],"Integrated":[72],"Gradients":[73],"based":[74],"token":[75],"attribution,":[76],"2)":[77],"outputs":[79],"Softmax":[82],"probabilities,":[83],"3)":[85],"internal":[87],"state":[88],"self-attention":[90],"fully-connected":[92],"layer":[93],"activations":[94],"for":[95],"signs":[96],"on":[99,122],"open-ended":[100],"question":[101],"answering":[102],"tasks.":[103],"Our":[104],"results":[105],"show":[106,154],"distributions":[109],"these":[111,131],"tend":[113],"to":[114,136,149],"differ":[115],"between":[116],"hallucinated":[117],"non-hallucinated":[119],"generations.":[120],"Building":[121],"insight,":[124],"train":[126],"binary":[127],"classifiers":[128,146],"use":[130],"as":[133],"input":[134],"features":[135],"classify":[137],"into":[140],"non-hallucinations.":[143],"These":[144],"hallucination":[145,159,165],"achieve":[147],"up":[148],"0.80":[150],"AUROC.":[151],"We":[152],"also":[153],"tokens":[156],"preceding":[157],"already":[161],"predict":[162],"subsequent":[164],"before":[167],"it":[168],"occurs.":[169]},"counts_by_year":[{"year":2025,"cited_by_count":8}],"updated_date":"2026-02-25T23:00:34.991745","created_date":"2025-10-10T00:00:00"}