{"id":"https://openalex.org/W4396723183","doi":"https://doi.org/10.1145/3589334.3645623","title":"KGQuiz: Evaluating the Generalization of Encoded Knowledge in Large Language Models","display_name":"KGQuiz: Evaluating the Generalization of Encoded Knowledge in Large Language Models","publication_year":2024,"publication_date":"2024-05-08","ids":{"openalex":"https://openalex.org/W4396723183","doi":"https://doi.org/10.1145/3589334.3645623"},"language":"en","primary_location":{"id":"doi:10.1145/3589334.3645623","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3589334.3645623","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Web Conference 2024","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100624456","display_name":"Yuyang Bai","orcid":"https://orcid.org/0000-0001-6261-4583"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuyang Bai","raw_affiliation_strings":["Xi'an Jiaotong University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0001-6261-4583","affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039166924","display_name":"Shangbin Feng","orcid":"https://orcid.org/0000-0002-4133-1987"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shangbin Feng","raw_affiliation_strings":["University of Washington, Seattle, USA"],"raw_orcid":"https://orcid.org/0000-0002-4133-1987","affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, USA","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I58610484"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012491878","display_name":"Vidhisha Balachandran","orcid":"https://orcid.org/0009-0009-0465-0098"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vidhisha Balachandran","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, USA"],"raw_orcid":"https://orcid.org/0009-0009-0465-0098","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047674498","display_name":"Zhaoxuan Tan","orcid":"https://orcid.org/0000-0001-8230-6238"},"institutions":[{"id":"https://openalex.org/I107639228","display_name":"University of Notre Dame","ror":"https://ror.org/00mkhxb43","country_code":"US","type":"education","lineage":["https://openalex.org/I107639228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhaoxuan Tan","raw_affiliation_strings":["University of Notre Dame, Notre Dame, USA"],"raw_orcid":"https://orcid.org/0000-0001-8230-6238","affiliations":[{"raw_affiliation_string":"University of Notre Dame, Notre Dame, USA","institution_ids":["https://openalex.org/I107639228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085113074","display_name":"Shiqi Lou","orcid":"https://orcid.org/0000-0003-4615-5481"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shiqi Lou","raw_affiliation_strings":["Xi'an Jiaotong University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0003-4615-5481","affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051747323","display_name":"Tianxing He","orcid":"https://orcid.org/0009-0008-6383-0307"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tianxing He","raw_affiliation_strings":["University of Washington, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0008-6383-0307","affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, USA","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I58610484"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5062910836","display_name":"Yulia Tsvetkov","orcid":"https://orcid.org/0000-0002-4634-7128"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I58610484","display_name":"Seattle University","ror":"https://ror.org/02jqc0m91","country_code":"US","type":"education","lineage":["https://openalex.org/I58610484"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yulia Tsvetkov","raw_affiliation_strings":["University of Washington, Seattle, USA"],"raw_orcid":"https://orcid.org/0000-0002-4634-7128","affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, USA","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I58610484"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2226","last_page":"2237"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.9763000011444092,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7232130765914917},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.7023887038230896},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5583848357200623},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45961353182792664},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.13139671087265015}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7232130765914917},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.7023887038230896},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5583848357200623},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45961353182792664},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.13139671087265015},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3589334.3645623","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3589334.3645623","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Web Conference 2024","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7599999904632568,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G2972134439","display_name":null,"funder_award_id":"HR001120C0124","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"},{"id":"https://openalex.org/G8085728410","display_name":null,"funder_award_id":"IIS2125201, IIS2203097","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"}],"funders":[{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":26,"referenced_works":["https://openalex.org/W1975879668","https://openalex.org/W2560674852","https://openalex.org/W2561529111","https://openalex.org/W2750779823","https://openalex.org/W2912500072","https://openalex.org/W2912924812","https://openalex.org/W2963339397","https://openalex.org/W2963829073","https://openalex.org/W3011594683","https://openalex.org/W3097986428","https://openalex.org/W3198455100","https://openalex.org/W3205810519","https://openalex.org/W3212511129","https://openalex.org/W4205450747","https://openalex.org/W4285294723","https://openalex.org/W4287111051","https://openalex.org/W4309674289","https://openalex.org/W4385569761","https://openalex.org/W4385569780","https://openalex.org/W4385570140","https://openalex.org/W4385570777","https://openalex.org/W4385573837","https://openalex.org/W4385573912","https://openalex.org/W4385574183","https://openalex.org/W4400064739","https://openalex.org/W4401042394"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W3162204513","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLMs)":[3],"demonstrate":[4,145],"remarkable":[5],"performance":[6,150,184],"on":[7,23,131],"knowledge-intensive":[8,68,138],"tasks,":[9,155],"suggesting":[10],"that":[11,146],"real-world":[12],"knowledge":[13,30,41,47,54,74,91,110,119,141,153,198,205],"is":[14,33,80],"encoded":[15],"in":[16,28,151,183],"their":[17,46,122],"model":[18],"parameters.":[19],"However,":[20],"besides":[21],"explorations":[22],"a":[24,51,67,81,114,176,201],"few":[25],"probing":[26],"tasks":[27,97,139],"limited":[29],"domains,":[31],"it":[32],"not":[34],"well":[35,45],"understood":[36],"how":[37,44],"to":[38,70,178,192],"evaluate":[39,125],"LLMs'":[40,118,197],"systematically":[42],"and":[43,56,93,108,121,128,140,158,187,190,195,207],"abilities":[48,76,120,199],"generalize,":[49],"across":[50,135,185,200],"spectrum":[52,203],"of":[53,77,95,117,204],"domains":[55,92,186,206],"progressively":[57],"complex":[58,162],"task":[59,188],"formats.":[60],"To":[61,112],"this":[62],"end,":[63],"we":[64,124],"propose":[65],"KGQuiz,":[66],"benchmark":[69,134],"comprehensively":[71],"investigate":[72],"the":[73,132,136],"generalization":[75],"LLMs.":[78],"KGQuiz":[79,133,174],"scalable":[82],"framework":[83],"constructed":[84],"from":[85],"triplet-based":[86],"knowledge,":[87],"which":[88],"covers":[89],"three":[90],"consists":[94],"five":[96,137],"with":[98],"increasing":[99],"complexity:":[100],"true-or-false,":[101],"multiple-choice":[102],"QA,":[103],"blank":[104],"filling,":[105],"factual":[106],"editing,":[107],"open-ended":[109],"generation.":[111],"gain":[113],"better":[115],"understanding":[116],"generalization,":[123],"10":[126],"open-source":[127],"black-box":[129],"LLMs":[130,147],"domains.":[142],"Extensive":[143],"experiments":[144],"achieve":[148],"impressive":[149],"straightforward":[152],"QA":[154],"while":[156],"settings":[157],"contexts":[159],"requiring":[160],"more":[161],"reasoning":[163],"or":[164],"employing":[165],"domain-specific":[166],"facts":[167],"still":[168],"present":[169],"significant":[170],"challenges.":[171],"We":[172],"envision":[173],"as":[175],"testbed":[177],"analyze":[179],"such":[180],"nuanced":[181],"variations":[182],"formats,":[189],"ultimately":[191],"understand,":[193],"evaluate,":[194],"improve":[196],"wide":[202],"tasks.":[208]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}