{"id":"https://openalex.org/W4404391920","doi":"https://doi.org/10.48550/arxiv.2411.07140","title":"Chinese SimpleQA: A Chinese Factuality Evaluation for Large Language Models","display_name":"Chinese SimpleQA: A Chinese Factuality Evaluation for Large Language Models","publication_year":2024,"publication_date":"2024-11-11","ids":{"openalex":"https://openalex.org/W4404391920","doi":"https://doi.org/10.48550/arxiv.2411.07140"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2411.07140","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.07140","pdf_url":"https://arxiv.org/pdf/2411.07140","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2411.07140","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103325403","display_name":"Yancheng He","orcid":"https://orcid.org/0009-0003-5078-0447","observed_orcids":["https://orcid.org/0009-0003-5078-0447"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Yancheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100725235","display_name":"Shilong Li","orcid":"https://orcid.org/0009-0005-7538-1754","observed_orcids":["https://orcid.org/0009-0005-7538-1754"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Shilong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115596116","display_name":"Jiaheng Liu","orcid":"https://orcid.org/0009-0001-6029-8991","observed_orcids":["https://orcid.org/0009-0001-6029-8991"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jiaheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100551756","display_name":"Yingshui Tan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tan, Yingshui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082417441","display_name":"Weixun Wang","orcid":"https://orcid.org/0000-0002-2727-8948","observed_orcids":["https://orcid.org/0000-0002-2727-8948"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Weixun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055090181","display_name":"Hui Huang","orcid":"https://orcid.org/0000-0001-9159-8455","observed_orcids":["https://orcid.org/0000-0001-9159-8455"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Hui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039569339","display_name":"Xingyuan Bu","orcid":"https://orcid.org/0000-0002-6445-4306","observed_orcids":["https://orcid.org/0000-0002-6445-4306"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bu, Xingyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058236062","display_name":"Hangyu Guo","orcid":"https://orcid.org/0009-0005-5744-8935","observed_orcids":["https://orcid.org/0009-0005-5744-8935"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Hangyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104156897","display_name":"Chengwei Hu","orcid":"https://orcid.org/0009-0009-5126-9767","observed_orcids":["https://orcid.org/0009-0009-5126-9767"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Chengwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051860912","display_name":"Boren Zheng","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Boren","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Lin, Zhuoran","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Zhuoran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061496726","display_name":"Xuepeng Liu","orcid":"https://orcid.org/0009-0007-3007-6793","observed_orcids":["https://orcid.org/0009-0007-3007-6793"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Xuepeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015083720","display_name":"Dekai Sun","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Dekai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Lin, Shirong","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Shirong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zheng, Zhicheng","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Zhicheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zhu, Xiaoyong","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Xiaoyong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113392255","display_name":"Wenbo Su","orcid":"https://orcid.org/0009-0009-3800-7543","observed_orcids":["https://orcid.org/0009-0009-3800-7543"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Su, Wenbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5050479679","display_name":"B. Zheng","orcid":"https://orcid.org/0000-0002-6544-429X","observed_orcids":["https://orcid.org/0000-0002-6544-429X","https://orcid.org/0009-0009-1601-4734","https://orcid.org/0000-0003-4308-3742","https://orcid.org/0000-0002-4037-6315"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng, Bo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9740999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9740999937057495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.009999999776482582,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.00800000037997961,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/factuality-evaluation","display_name":"factuality evaluation","score":0.824999988079071},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9300000071525574},{"id":"https://openalex.org/keywords/chinese-language","display_name":"Chinese language","score":0.3959999978542328},{"id":"https://openalex.org/keywords/question-answering-benchmark","display_name":"question answering benchmark","score":0.6340000033378601},{"id":"https://openalex.org/keywords/factual-accuracy","display_name":"factual accuracy","score":0.5059999823570251},{"id":"https://openalex.org/keywords/llm-benchmark","display_name":"LLM benchmark","score":0.6740000247955322}],"concepts":[{"id":"https://openalex.org/C3018428822","wikidata":"https://www.wikidata.org/wiki/Q7850","display_name":"Chinese language","level":2,"score":0.6466341018676758},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5549677610397339},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5241846442222595},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.42970818281173706},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37277519702911377},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.11755362153053284}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2411.07140","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.07140","pdf_url":"https://arxiv.org/pdf/2411.07140","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2411.07140","doi":"https://doi.org/10.48550/arxiv.2411.07140","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.07140","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2411.07140","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.07140","pdf_url":"https://arxiv.org/pdf/2411.07140","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404391920.pdf","grobid_xml":"https://content.openalex.org/works/W4404391920.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4405713880","https://openalex.org/W4416068860","https://openalex.org/W4404396608","https://openalex.org/W4396820591","https://openalex.org/W4416056056","https://openalex.org/W4416452639","https://openalex.org/W7017787997","https://openalex.org/W4392492460","https://openalex.org/W4416035248","https://openalex.org/W7223358976"],"abstract_inverted_index":{"New":[0],"LLM":[1],"evaluation":[2,124],"benchmarks":[3],"are":[4,88,103],"important":[5],"to":[6,29,37,78,142],"align":[7],"with":[8,66],"the":[9,24,31,59,85,99,107,126,140,145,154],"rapid":[10],"development":[11],"of":[12,34,129,149,156],"Large":[13],"Language":[14],"Models":[15],"(LLMs).":[16],"In":[17],"this":[18],"work,":[19],"we":[20,56,71,120,133],"present":[21],"Chinese":[22,27,42,60,118,136,146],"SimpleQA,":[23,98,119],"first":[25],"comprehensive":[26,74,123],"benchmark":[28],"evaluate":[30],"factuality":[32,127,147],"ability":[33],"language":[35,61],"models":[36,151],"answer":[38],"short":[39],"questions,":[40],"and":[41,82,90,101,106,152],"SimpleQA":[43,137],"mainly":[44],"has":[45],"five":[46],"properties":[47],"(i.e.,":[48],"Chinese,":[49],"Diverse,":[50],"High-quality,":[51],"Static,":[52],"Easy-to-evaluate).":[53],"Specifically,":[54],"first,":[55],"focus":[57],"on":[58,113,117,125],"over":[62,94],"6":[63],"major":[64],"topics":[65],"99":[67],"diverse":[68],"subtopics.":[69],"Second,":[70],"conduct":[72],"a":[73,122],"quality":[75],"control":[76],"process":[77,109],"achieve":[79],"high-quality":[80],"questions":[81,100],"answers,":[83],"where":[84],"reference":[86],"answers":[87,102],"static":[89],"cannot":[91],"be":[92],"changed":[93],"time.":[95],"Third,":[96],"following":[97],"very":[104],"short,":[105],"grading":[108],"is":[110],"easy-to-evaluate":[111],"based":[112],"OpenAI":[114],"API.":[115],"Based":[116],"perform":[121],"abilities":[128,148],"existing":[130],"LLMs.":[131],"Finally,":[132],"hope":[134],"that":[135],"could":[138],"guide":[139],"developers":[141],"better":[143],"understand":[144],"their":[150],"facilitate":[153],"growth":[155],"foundation":[157],"models.":[158]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-11-01T11:40:01.658920","created_date":"2024-11-15T00:00:00"}