{"id":"https://openalex.org/W4389520749","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.557","title":"SelfCheckGPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models","display_name":"SelfCheckGPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389520749","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.557"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.emnlp-main.557","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.557","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-main.557","pdf_url":"https://aclanthology.org/2023.emnlp-main.557.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.emnlp-main.557.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024447363","display_name":"Potsawee Manakul","orcid":"https://orcid.org/0000-0001-7108-8626","observed_orcids":["https://orcid.org/0000-0001-7108-8626"]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Potsawee Manakul","raw_affiliation_strings":["ALTA Institute, Department of Engineering, University of Cambridge","Pavia, Italy","born in Naples, Italy, in 1882, and died in Paris, France, in 1944"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ALTA Institute, Department of Engineering, University of Cambridge","institution_ids":["https://openalex.org/I241749"]},{"raw_affiliation_string":"Pavia, Italy","institution_ids":[]},{"raw_affiliation_string":"born in Naples, Italy, in 1882, and died in Paris, France, in 1944","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011907463","display_name":"Adian Liusie","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Adian Liusie","raw_affiliation_strings":["ALTA Institute, Department of Engineering, University of Cambridge","Pavia, Italy","born in Naples, Italy, in 1882, and died in Paris, France, in 1944"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ALTA Institute, Department of Engineering, University of Cambridge","institution_ids":["https://openalex.org/I241749"]},{"raw_affiliation_string":"Pavia, Italy","institution_ids":[]},{"raw_affiliation_string":"born in Naples, Italy, in 1882, and died in Paris, France, in 1944","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5050766679","display_name":"Mark Gales","orcid":"https://orcid.org/0000-0002-5311-8219","observed_orcids":["https://orcid.org/0000-0002-5311-8219"]},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mark Gales","raw_affiliation_strings":["ALTA Institute, Department of Engineering, University of Cambridge","Pavia, Italy","born in Naples, Italy, in 1882, and died in Paris, France, in 1944"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ALTA Institute, Department of Engineering, University of Cambridge","institution_ids":["https://openalex.org/I241749"]},{"raw_affiliation_string":"Pavia, Italy","institution_ids":[]},{"raw_affiliation_string":"born in Naples, Italy, in 1882, and died in Paris, France, in 1944","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I241749"],"apc_list":null,"apc_paid":null,"fwci":162.7566,"has_fulltext":true,"cited_by_count":1437,"citation_normalized_percentile":{"value":0.99973794,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"9004","last_page":"9017"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6776999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6776999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.22990000247955322,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.03849999979138374,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hallucination-detection","display_name":"hallucination detection","score":0.9700000286102295},{"id":"https://openalex.org/keywords/fact-checking","display_name":"fact-checking","score":0.7139999866485596},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8560000061988831},{"id":"https://openalex.org/keywords/black-box-modeling","display_name":"black-box models","score":0.8209999799728394},{"id":"https://openalex.org/keywords/gpt-3","display_name":"GPT-3","score":0.8169999718666077}],"concepts":[{"id":"https://openalex.org/C2911011789","wikidata":"https://www.wikidata.org/wiki/Q130741","display_name":"Hallucinating","level":2,"score":0.9820280075073242},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7362278699874878},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.622717022895813},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.5926380157470703},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.5603088140487671},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.5526127815246582},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5514890551567078},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.5414992570877075},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.5384971499443054},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5343249440193176},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.4998776912689209},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.4906269609928131},{"id":"https://openalex.org/C2780813799","wikidata":"https://www.wikidata.org/wiki/Q3274237","display_name":"Zero (linguistics)","level":2,"score":0.4625721573829651},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4445892572402954},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.44204625487327576},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.16691303253173828},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.11598321795463562},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.11381605267524719},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10738241672515869},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.18653/v1/2023.emnlp-main.557","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.557","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-main.557","pdf_url":"https://aclanthology.org/2023.emnlp-main.557.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2303.08896","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2303.08896","pdf_url":"https://arxiv.org/pdf/2303.08896","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:www.repository.cam.ac.uk:1810/358475","doi":null,"is_oa":true,"landing_page_url":"https://www.repository.cam.ac.uk/handle/1810/358475","pdf_url":null,"source":{"id":"https://openalex.org/S4306401777","display_name":"Apollo (University of Cambridge)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I241749","host_organization_name":"University of Cambridge","host_organization_lineage":["https://openalex.org/I241749"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference Object"},{"id":"doi:10.48550/arxiv.2303.08896","doi":"https://doi.org/10.48550/arxiv.2303.08896","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2303.08896","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.17863/cam.102103","doi":"https://doi.org/10.17863/cam.102103","is_oa":true,"landing_page_url":"https://doi.org/10.17863/cam.102103","pdf_url":null,"source":{"id":"https://openalex.org/S7407050737","display_name":"Apollo","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Other"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.emnlp-main.557","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.557","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-main.557","pdf_url":"https://aclanthology.org/2023.emnlp-main.557.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320320273","display_name":"University of Cambridge","ror":"https://ror.org/013meh722"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389520749.pdf","grobid_xml":"https://content.openalex.org/works/W4389520749.grobid-xml"},"referenced_works_count":39,"referenced_works":["https://openalex.org/W1444168786","https://openalex.org/W2053154970","https://openalex.org/W2305610840","https://openalex.org/W2606964149","https://openalex.org/W2607892599","https://openalex.org/W2902421513","https://openalex.org/W2948947170","https://openalex.org/W2963748441","https://openalex.org/W2965373594","https://openalex.org/W2981456979","https://openalex.org/W2981852735","https://openalex.org/W2998230451","https://openalex.org/W3015468748","https://openalex.org/W3021338988","https://openalex.org/W3030163527","https://openalex.org/W3034808961","https://openalex.org/W3123791752","https://openalex.org/W3153712677","https://openalex.org/W3155332104","https://openalex.org/W3155807546","https://openalex.org/W3158419028","https://openalex.org/W3196268181","https://openalex.org/W3211686893","https://openalex.org/W3212554815","https://openalex.org/W4221161695","https://openalex.org/W4224308101","https://openalex.org/W4229005866","https://openalex.org/W4285225959","https://openalex.org/W4285265130","https://openalex.org/W4285429195","https://openalex.org/W4309674289","https://openalex.org/W4321594373","https://openalex.org/W4322718191","https://openalex.org/W4361230777","https://openalex.org/W4386576669","https://openalex.org/W4389519449","https://openalex.org/W4389519598","https://openalex.org/W4392669872","https://openalex.org/W4401043863"],"related_works":["https://openalex.org/W6948021977","https://openalex.org/W4401043402","https://openalex.org/W4416354421","https://openalex.org/W4413423557","https://openalex.org/W7201835280","https://openalex.org/W7083678916","https://openalex.org/W4402632047","https://openalex.org/W4415125767","https://openalex.org/W4392617635","https://openalex.org/W7066025966"],"abstract_inverted_index":{"Generative":[0],"Large":[1],"Language":[2],"Models":[3],"(LLMs)":[4],"such":[5,56],"as":[6,57],"GPT-3":[7,143],"are":[8,23,63,115,130],"capable":[9],"of":[10,19,87,109,158,178],"generating":[11],"highly":[12],"fluent":[13],"responses":[14,86,114,129],"to":[15,25,44,83,117,132,144,183,209],"a":[16,75,91,110],"wide":[17],"variety":[18],"user":[20],"prompts.However,":[21],"LLMs":[22],"known":[24],"hallucinate":[26],"facts":[27],"and":[28,120,134,153,169,172,186,200],"make":[29],"non-factual":[30,168],"statements":[31],"which":[32],"can":[33,80],"undermine":[34],"trust":[35],"in":[36,90,176,196,204],"their":[37],"output.Existing":[38],"fact-checking":[39],"approaches":[40],"either":[41],"require":[42],"access":[43],"the":[45,85,100,150,156,159],"output":[46],"probability":[47],"distribution":[48],"(which":[49],"may":[50],"not":[51],"be":[52,81,118],"available":[53],"for":[54,124],"systems":[55],"ChatGPT)":[58],"or":[59],"external":[60,97],"databases":[61],"that":[62,79,103,163,188],"interfaced":[64],"via":[65],"separate,":[66],"often":[67],"complex,":[68],"modules.In":[69],"this":[70,139],"work,":[71],"we":[72],"propose":[73],"\"SelfCheckGPT\",":[74],"simple":[76,101],"sampling-based":[77],"approach":[78,140,182,190],"used":[82],"fact-check":[84],"black-box":[88],"models":[89],"zero-resource":[92],"fashion,":[93],"i.e.":[94],"without":[95],"an":[96,105],"database.SelfCheckGPT":[98],"leverages":[99],"idea":[102],"if":[104],"LLM":[106],"has":[107,191],"knowledge":[108],"given":[111],"concept,":[112],"sampled":[113,128],"likely":[116,131],"similar":[119],"contain":[121],"consistent":[122],"facts.However,":[123],"hallucinated":[125],"facts,":[126],"stochastically":[127],"diverge":[133],"contradict":[135],"one":[136],"another.We":[137],"investigate":[138],"by":[141],"using":[142],"generate":[145],"passages":[146,175],"about":[147],"individuals":[148],"from":[149],"WikiBio":[151],"dataset,":[152],"manually":[154],"annotate":[155],"factuality":[157,206],"generated":[160],"passages.We":[161],"demonstrate":[162],"SelfCheck-GPT":[164],"can:":[165],"i)":[166],"detect":[167],"factual":[170],"sentences;":[171],"ii)":[173],"rank":[174],"terms":[177],"factuality.We":[179],"compare":[180],"our":[181,189],"several":[184],"baselines":[185],"show":[187],"considerably":[192],"higher":[193,201],"AUC-PR":[194],"scores":[195,203],"sentence-level":[197],"hallucination":[198],"detection":[199],"correlation":[202],"passage-level":[205],"assessment":[207],"compared":[208],"grey-box":[210],"methods.":[211]},"counts_by_year":[{"year":2026,"cited_by_count":672},{"year":2025,"cited_by_count":415},{"year":2024,"cited_by_count":262},{"year":2023,"cited_by_count":87},{"year":2022,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2023-12-11T00:00:00"}