{"id":"https://openalex.org/W4410636335","doi":"https://doi.org/10.1145/3701716.3715586","title":"Can Large Language Models Be a Good Evaluator for Review-based Product Question Answering?","display_name":"Can Large Language Models Be a Good Evaluator for Review-based Product Question Answering?","publication_year":2025,"publication_date":"2025-05-08","ids":{"openalex":"https://openalex.org/W4410636335","doi":"https://doi.org/10.1145/3701716.3715586"},"language":"en","primary_location":{"id":"doi:10.1145/3701716.3715586","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3701716.3715586","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3701716.3715586","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the ACM on Web Conference 2025","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3701716.3715586","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045801173","display_name":"Tony Danhui Huang","orcid":"https://orcid.org/0000-0002-1356-9023"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Tony Danhui Huang","raw_affiliation_strings":["School of Computing Technologies, RMIT University, Melbourne, Victoria, Australia"],"affiliations":[{"raw_affiliation_string":"School of Computing Technologies, RMIT University, Melbourne, Victoria, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039040673","display_name":"Yongli Ren","orcid":"https://orcid.org/0000-0002-3137-9653"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Yongli Ren","raw_affiliation_strings":["School of Computing Technologies, RMIT University, Melbourne, Victoria, Australia"],"affiliations":[{"raw_affiliation_string":"School of Computing Technologies, RMIT University, Melbourne, Victoria, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021983750","display_name":"Xiuzhen Zhang","orcid":"https://orcid.org/0000-0001-5558-3790"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Xiuzhen Zhang","raw_affiliation_strings":["School of Computing Technologies, RMIT University, Melbourne, Victoria, Australia"],"affiliations":[{"raw_affiliation_string":"School of Computing Technologies, RMIT University, Melbourne, Victoria, Australia","institution_ids":["https://openalex.org/I82951845"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5045801173"],"corresponding_institution_ids":["https://openalex.org/I82951845"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.16062901,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1019","last_page":"1023"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9768000245094299,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.8145427107810974},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7613087892532349},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5991496443748474},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.48626887798309326},{"id":"https://openalex.org/keywords/product","display_name":"Product (mathematics)","score":0.4785747826099396},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.43289715051651},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.36031824350357056},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09561043977737427}],"concepts":[{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.8145427107810974},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7613087892532349},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5991496443748474},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48626887798309326},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.4785747826099396},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.43289715051651},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.36031824350357056},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09561043977737427},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3701716.3715586","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3701716.3715586","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3701716.3715586","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the ACM on Web Conference 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3701716.3715586","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3701716.3715586","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3701716.3715586","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the ACM on Web Conference 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4410636335.pdf","grobid_xml":"https://content.openalex.org/works/W4410636335.grobid-xml"},"referenced_works_count":5,"referenced_works":["https://openalex.org/W2084335597","https://openalex.org/W2243869100","https://openalex.org/W4385569780","https://openalex.org/W4389519920","https://openalex.org/W4402351491"],"related_works":["https://openalex.org/W2384605597","https://openalex.org/W2387743295","https://openalex.org/W3082787378","https://openalex.org/W2136007095","https://openalex.org/W2366230879","https://openalex.org/W3208425359","https://openalex.org/W2349927912","https://openalex.org/W3159777597","https://openalex.org/W4212839359","https://openalex.org/W2115758952"],"abstract_inverted_index":{"Large":[0],"language":[1,11],"models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"impressive":[6],"performance":[7,203],"on":[8,32,66,168],"evaluating":[9],"natural":[10],"generation":[12],"tasks.":[13],"Review-based":[14],"Product":[15],"Question":[16,25],"Answering":[17,26],"(RPQA)":[18],"evaluation,":[19,28,60],"which":[20],"is":[21,71,209],"a":[22,72,132,165],"domain":[23,109],"knowledge-intensive":[24,110],"(QA)":[27],"still":[29],"largely":[30],"relies":[31],"lexicon-based":[33],"metrics":[34,40,50],"and":[35,146,173],"frozen":[36],"embedding-based":[37],"metrics.":[38],"Those":[39],"fail":[41],"if":[42],"reference":[43],"answers":[44],"are":[45],"absent.":[46],"Despite":[47],"some":[48],"model-based":[49],"being":[51],"learned":[52],"from":[53],"in-domain":[54],"data":[55],"for":[56,126,152],"the":[57,96,119,143,147,191,195,202,205],"RPQA":[58,111,127,153],"task":[59],"little":[61],"research":[62],"has":[63,77],"been":[64,78],"done":[65],"using":[67,95],"LLMs.":[68,175],"Chain-of-Thought":[69],"(CoT)":[70],"state-of-the-art":[73],"prompting":[74,161,189],"method":[75],"that":[76,179],"proposed":[79],"to":[80,83],"induce":[81],"LLMs":[82,123],"solve":[84],"complex":[85],"problems":[86],"efficiently.":[87],"The":[88],"CoT":[89,144,160],"reasoning":[90],"steps":[91],"can":[92,182],"be":[93,106],"resolved":[94],"internal":[97,103],"knowledge":[98,104],"of":[99,121,204],"LLMs,":[100,170],"but":[101],"this":[102,115],"may":[105],"insufficient":[107],"in":[108,194,201],"evaluation":[112],"settings.":[113],"In":[114],"work,":[116],"we":[117,130,197],"explore":[118],"feasibility":[120],"leveraging":[122],"as":[124,158],"evaluators":[125,181],"evaluation.":[128,154],"Specifically,":[129],"design":[131],"structured":[133],"prompt":[134],"template":[135],"with":[136],"one-shot":[137],"or":[138],"few-shot":[139],"in-context":[140],"learning,":[141],"incorporating":[142],"mechanism":[145],"System":[148],"2":[149],"thinking":[150],"process":[151],"We":[155,163],"call":[156],"it":[157],"ExampLe-Enhanced":[159],"(ELECT).":[162],"conduct":[164],"comprehensive":[166],"study":[167],"various":[169],"including":[171],"evaluation-oriented":[172],"general-purpose":[174],"Experimental":[176],"results":[177],"show":[178],"LLM-based":[180,206],"effectively":[183],"evaluate":[184],"RPQA.":[185],"Upon":[186],"integrating":[187],"ELECT":[188],"into":[190],"demonstration":[192],"examples":[193],"prompt,":[196],"observe":[198],"an":[199],"improvement":[200],"evaluator.":[207],"Code":[208],"publicly":[210],"available":[211],"at":[212],"https://github.com/tonydhuang/elect-prompting.":[213]},"counts_by_year":[],"updated_date":"2026-03-10T16:38:18.471706","created_date":"2025-10-10T00:00:00"}