{"id":"https://openalex.org/W7124434995","doi":"https://doi.org/10.7717/peerj-cs.3446","title":"A topic-aware evaluation of ChatGPT\u2019s semantic alignment with community answers using BERTScore and BERTopic","display_name":"A topic-aware evaluation of ChatGPT\u2019s semantic alignment with community answers using BERTScore and BERTopic","publication_year":2026,"publication_date":"2026-01-16","ids":{"openalex":"https://openalex.org/W7124434995","doi":"https://doi.org/10.7717/peerj-cs.3446"},"language":"en","primary_location":{"id":"doi:10.7717/peerj-cs.3446","doi":"https://doi.org/10.7717/peerj-cs.3446","is_oa":true,"landing_page_url":"https://doi.org/10.7717/peerj-cs.3446","pdf_url":"https://peerj.com/articles/cs-3446.pdf","source":{"id":"https://openalex.org/S4210178049","display_name":"PeerJ Computer Science","issn_l":"2376-5992","issn":["2376-5992"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","jufo-1","ki-jl-1","norway-1","poland-70","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320104","host_organization_name":"PeerJ, Inc.","host_organization_lineage":["https://openalex.org/P4310320104"],"host_organization_lineage_names":["PeerJ, Inc."],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PeerJ Computer Science","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://peerj.com/articles/cs-3446.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010624503","display_name":"Mashael M. Alsulami","orcid":"https://orcid.org/0000-0003-2298-8278","observed_orcids":["https://orcid.org/0000-0003-2298-8278"]},"institutions":[{"id":"https://openalex.org/I179331831","display_name":"Taif University","ror":"https://ror.org/014g1a453","country_code":"SA","type":"education","lineage":["https://openalex.org/I179331831"]}],"countries":["SA"],"is_corresponding":true,"raw_author_name":"Mashael M. Alsulami","raw_affiliation_strings":["Department of Information Technology , College of Computers and Information Technology , Taif University , Taif , Makkah , Saudi Arabia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Information Technology , College of Computers and Information Technology , Taif University , Taif , Makkah , Saudi Arabia","institution_ids":["https://openalex.org/I179331831"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5010624503"],"corresponding_institution_ids":["https://openalex.org/I179331831"],"apc_list":{"value":1395,"currency":"USD","value_usd":1395},"apc_paid":{"value":1395,"currency":"USD","value_usd":1395},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.04533686,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"12","issue":null,"first_page":"e3446","last_page":"e3446"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.002400000113993883,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.002099999925121665,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/chatgpt","display_name":"ChatGPT","score":0.9950000047683716},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8149999976158142},{"id":"https://openalex.org/keywords/open-domain-question-answering","display_name":"open-domain question answering","score":0.9739999771118164},{"id":"https://openalex.org/keywords/semantic-alignment","display_name":"semantic alignment","score":0.9459999799728394},{"id":"https://openalex.org/keywords/yahoo-answers","display_name":"Yahoo! Answers","score":0.9610000252723694},{"id":"https://openalex.org/keywords/bertscore","display_name":"BERTScore","score":0.8320000171661377},{"id":"https://openalex.org/keywords/bertopic","display_name":"BERTopic","score":0.9990000128746033},{"id":"https://openalex.org/keywords/conversational-ai","display_name":"conversational AI","score":0.746999979019165}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7653999924659729},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6123999953269958},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5543000102043152},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.46779999136924744},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.4616999924182892},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.44130000472068787},{"id":"https://openalex.org/C170133592","wikidata":"https://www.wikidata.org/wiki/Q1806883","display_name":"Latent semantic analysis","level":2,"score":0.4262999892234802},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.4075999855995178},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4020000100135803},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.4018000066280365},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.3961000144481659},{"id":"https://openalex.org/C2777946921","wikidata":"https://www.wikidata.org/wiki/Q7449044","display_name":"Semantic analysis (machine learning)","level":2,"score":0.3531999886035919},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.31869998574256897},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.2906000018119812},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2809000015258789},{"id":"https://openalex.org/C90312973","wikidata":"https://www.wikidata.org/wiki/Q7449052","display_name":"Semantic data model","level":2,"score":0.2694999873638153},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2630999982357025},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.26260000467300415},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2583000063896179},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.2500999867916107}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.7717/peerj-cs.3446","doi":"https://doi.org/10.7717/peerj-cs.3446","is_oa":true,"landing_page_url":"https://doi.org/10.7717/peerj-cs.3446","pdf_url":"https://peerj.com/articles/cs-3446.pdf","source":{"id":"https://openalex.org/S4210178049","display_name":"PeerJ Computer Science","issn_l":"2376-5992","issn":["2376-5992"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","jufo-1","ki-jl-1","norway-1","poland-70","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320104","host_organization_name":"PeerJ, Inc.","host_organization_lineage":["https://openalex.org/P4310320104"],"host_organization_lineage_names":["PeerJ, Inc."],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PeerJ Computer Science","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:c031fff926954781964a91fa82922819","doi":null,"is_oa":true,"landing_page_url":"https://doaj.org/article/c031fff926954781964a91fa82922819","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"PeerJ Computer Science, Vol 12, p e3446 (2026)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.7717/peerj-cs.3446","doi":"https://doi.org/10.7717/peerj-cs.3446","is_oa":true,"landing_page_url":"https://doi.org/10.7717/peerj-cs.3446","pdf_url":"https://peerj.com/articles/cs-3446.pdf","source":{"id":"https://openalex.org/S4210178049","display_name":"PeerJ Computer Science","issn_l":"2376-5992","issn":["2376-5992"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["cwts-core","doaj","jufo-1","ki-jl-1","norway-1","poland-70","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310320104","host_organization_name":"PeerJ, Inc.","host_organization_lineage":["https://openalex.org/P4310320104"],"host_organization_lineage_names":["PeerJ, Inc."],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PeerJ Computer Science","raw_type":"journal-article"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"display_name":"Quality education","score":0.8195233345031738,"id":"https://metadata.un.org/sdg/4"}],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320323722","display_name":"Taif University","ror":"https://ror.org/014g1a453"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7124434995.pdf","grobid_xml":"https://content.openalex.org/works/W7124434995.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W1987111416","https://openalex.org/W2786672974","https://openalex.org/W2883730939","https://openalex.org/W2896457183","https://openalex.org/W2936695845","https://openalex.org/W2965373594","https://openalex.org/W2965783570","https://openalex.org/W3195577433","https://openalex.org/W4221142221","https://openalex.org/W4226278401","https://openalex.org/W4288253152","https://openalex.org/W4292779060","https://openalex.org/W4295290201","https://openalex.org/W4320854854","https://openalex.org/W4327810158","https://openalex.org/W4381663758","https://openalex.org/W4385569911","https://openalex.org/W4388132131","https://openalex.org/W4389766097","https://openalex.org/W4391113927","https://openalex.org/W4396832043","https://openalex.org/W4406152279","https://openalex.org/W4406260314","https://openalex.org/W4406261343","https://openalex.org/W4407736486","https://openalex.org/W4407736504"],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1,151],"models":[2],"are":[3,86],"increasingly":[4],"deployed":[5],"in":[6,32,154],"everyday":[7],"question-answering":[8],"tasks,":[9],"yet":[10],"their":[11],"performance":[12,61,131],"across":[13,62,133],"diverse,":[14],"real-world":[15,155],"queries":[16],"remains":[17],"underexplored.":[18],"This":[19,139],"study":[20,140],"evaluates":[21],"the":[22,42,160],"semantic":[23,90],"alignment":[24,113],"of":[25,66,109,149,162],"ChatGPT\u2019s":[26,58],"responses":[27,94],"with":[28,114],"human-selected":[29],"best":[30],"answers":[31],"an":[33],"open-domain":[34],"question":[35],"answering":[36],"(QA)":[37],"setting,":[38],"using":[39],"data":[40],"from":[41,47,75],"Yahoo!":[43],"Answers":[44],"platform.":[45],"Distinct":[46],"prior":[48],"research":[49],"that":[50,101,120,146],"centers":[51],"on":[52,125],"domain-specific":[53],"datasets,":[54],"this":[55],"work":[56],"explores":[57],"general-purpose":[59],"QA":[60,156],"a":[63,105,142],"wide":[64],"range":[65],"topics.":[67,138],"BERTopic":[68],"is":[69],"used":[70],"to":[71,88],"extract":[72],"latent":[73],"themes":[74],"500":[76],"diverse":[77],"full-question":[78],"samples,":[79],"and":[80,95,127,158,165],"BERTScore":[81],"metrics":[82],"(precision,":[83],"recall,":[84],"F1)":[85],"applied":[87],"quantify":[89],"similarity":[91],"between":[92],"ChatGPT-generated":[93],"top-rated":[96],"community":[97],"answers.":[98],"Results":[99],"indicate":[100],"ChatGPT":[102,121],"(GPT-3.5-turbo)":[103],"achieves":[104],"strong":[106],"average":[107],"F1-score":[108],"0.824,":[110],"reflecting":[111],"high":[112],"human":[115],"judgments.":[116],"Topic-level":[117],"analysis":[118],"reveals":[119],"performs":[122],"particularly":[123],"well":[124],"factual":[126],"encyclopedia-style":[128],"questions,":[129],"while":[130],"varies":[132],"more":[134,163],"subjective":[135],"or":[136],"open-ended":[137],"introduces":[141],"topic-sensitive":[143],"evaluation":[144],"framework":[145],"enhances":[147],"understanding":[148],"large":[150],"model":[152],"behavior":[153],"scenarios":[157],"supports":[159],"development":[161],"effective":[164],"explainable":[166],"conversational":[167],"artificial":[168],"intelligence":[169],"(AI)":[170],"systems.":[171]},"counts_by_year":[],"updated_date":"2026-10-14T16:39:01.820549","created_date":"2026-01-17T00:00:00"}