{"id":"https://openalex.org/W4412396441","doi":"https://doi.org/10.1145/3726302.3729895","title":"An Empirical Study of Evaluating Long-form Question Answering","display_name":"An Empirical Study of Evaluating Long-form Question Answering","publication_year":2025,"publication_date":"2025-07-13","ids":{"openalex":"https://openalex.org/W4412396441","doi":"https://doi.org/10.1145/3726302.3729895"},"language":"en","primary_location":{"id":"doi:10.1145/3726302.3729895","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3729895","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3729895","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3729895","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5118977189","display_name":"Ning Xian","orcid":null},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ning Xian","raw_affiliation_strings":["Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0004-1220-3021","affiliations":[{"raw_affiliation_string":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210090176","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006971161","display_name":"Yixing Fan","orcid":"https://orcid.org/0000-0003-4317-2702"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yixing Fan","raw_affiliation_strings":["Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-4317-2702","affiliations":[{"raw_affiliation_string":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210090176","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009898523","display_name":"Ruqing Zhang","orcid":"https://orcid.org/0000-0003-4294-2541"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruqing Zhang","raw_affiliation_strings":["Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-4294-2541","affiliations":[{"raw_affiliation_string":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210090176","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031439294","display_name":"Maarten de Rijke","orcid":"https://orcid.org/0000-0002-1086-0202"},"institutions":[{"id":"https://openalex.org/I4210135670","display_name":"Amsterdam University of the Arts","ror":"https://ror.org/04dde1554","country_code":"NL","type":"education","lineage":["https://openalex.org/I4210135670"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maarten de Rijke","raw_affiliation_strings":["University of Amsterdam, Amsterdam, Netherlands"],"raw_orcid":"https://orcid.org/0000-0002-1086-0202","affiliations":[{"raw_affiliation_string":"University of Amsterdam, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210135670","https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5088621320","display_name":"Jiafeng Guo","orcid":"https://orcid.org/0000-0002-9509-8674"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210090176","display_name":"Institute of Computing Technology","ror":"https://ror.org/0090r4d87","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210090176"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiafeng Guo","raw_affiliation_strings":["Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9509-8674","affiliations":[{"raw_affiliation_string":"Institute of Computing Technology, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210090176","https://openalex.org/I19820366"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1141","last_page":"1151"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9828000068664551,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.7701254487037659},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5588454604148865},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4693884253501892},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3704155683517456},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.32631629705429077},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.22552719712257385},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.16747671365737915}],"concepts":[{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.7701254487037659},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5588454604148865},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4693884253501892},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3704155683517456},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.32631629705429077},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.22552719712257385},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.16747671365737915}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3726302.3729895","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3729895","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3729895","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3726302.3729895","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3729895","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3726302.3729895","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2289169492","display_name":"Fairness and Intersectional Non-Discrimination in Human Recommendation","funder_award_id":"101070212","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G2526079209","display_name":null,"funder_award_id":"2023YFA1011602","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"},{"id":"https://openalex.org/G5550786340","display_name":"Hybrid Intelligence (HI): augmenting human intellect","funder_award_id":"024.004.022","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"},{"id":"https://openalex.org/G6280615682","display_name":"Low Resource Chat-based Conversational Intelligence","funder_award_id":"NWA.1389.20.183","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"},{"id":"https://openalex.org/G6640760388","display_name":"Trustworthy AI-based Systems for Sustainable Growth","funder_award_id":"KICH3.LTP.20.006","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"},{"id":"https://openalex.org/G6969014612","display_name":null,"funder_award_id":"62372431, 62472408 and 62441229","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"},{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"},{"id":"https://openalex.org/F4320334322","display_name":"HORIZON EUROPE Framework Programme","ror":null},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412396441.pdf","grobid_xml":"https://content.openalex.org/works/W4412396441.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W1498990157","https://openalex.org/W1593271688","https://openalex.org/W1982897610","https://openalex.org/W2950681488","https://openalex.org/W2970785793","https://openalex.org/W3017330897","https://openalex.org/W3034716087","https://openalex.org/W3104976898","https://openalex.org/W3169841173","https://openalex.org/W4226101361","https://openalex.org/W4284689799","https://openalex.org/W4309674289","https://openalex.org/W4385570371","https://openalex.org/W4385571232","https://openalex.org/W4385571412","https://openalex.org/W4385573898","https://openalex.org/W4385734111","https://openalex.org/W4385734161","https://openalex.org/W4386841309","https://openalex.org/W4389518874","https://openalex.org/W4389519239","https://openalex.org/W4389519254","https://openalex.org/W4389519598","https://openalex.org/W4392669905","https://openalex.org/W4393161118","https://openalex.org/W4401043863","https://openalex.org/W4401044057","https://openalex.org/W4402667158","https://openalex.org/W4402671800","https://openalex.org/W4404782181","https://openalex.org/W4404986534"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2384605597","https://openalex.org/W2387743295","https://openalex.org/W3082787378","https://openalex.org/W2136007095","https://openalex.org/W2366230879","https://openalex.org/W3208425359","https://openalex.org/W2349927912"],"abstract_inverted_index":{"Long-form":[0],"question":[1,206],"answering":[2],"(LFQA)":[3],"aims":[4],"to":[5,9,95],"generate":[6],"lengthy":[7],"answers":[8,47,118],"complex":[10],"questions.":[11],"This":[12],"scenario":[13],"presents":[14],"great":[15],"flexibility":[16],"as":[17,19,79],"well":[18],"significant":[20],"challenges":[21],"for":[22,45,82,196,203],"evaluation.":[23],"Most":[24],"evaluations":[25,44],"rely":[26],"on":[27,32,130,135,188],"deterministic":[28],"metrics":[29,77,93,147,157],"that":[30,163],"depend":[31],"string":[33],"or":[34],"n-gram":[35],"matching,":[36],"while":[37],"the":[38,65,88,101,142,153,164,168,171,177,197],"reliability":[39],"of":[40,60,90,105,132,144,167,173,199],"large":[41,122,200],"language":[42,123,201],"model-based":[43],"long-form":[46,61,117,205],"remains":[48],"relatively":[49],"unexplored.":[50],"We":[51,111,161],"address":[52],"this":[53,186],"gap":[54],"by":[55,120,148],"conducting":[56],"an":[57],"in-depth":[58],"study":[59],"answer":[62],"evaluation":[63,76,92,107,129,146,179,183],"with":[64],"following":[66],"research":[67],"questions:":[68],"(i)":[69],"To":[70],"what":[71],"extent":[72],"do":[73],"existing":[74,91,106],"automatic":[75,145,178],"serve":[78],"a":[80,127],"substitute":[81],"human":[83,96,128,159],"evaluations?":[84,97],"(ii)":[85],"What":[86],"are":[87,212],"limitations":[89],"compared":[94],"(iii)":[98],"How":[99],"can":[100,175],"effectiveness":[102],"and":[103,115,125,137,158,170,210],"robustness":[104],"methods":[108],"be":[109],"improved?":[110],"collect":[112],"5,236":[113],"factoid":[114],"non-factoid":[116],"generated":[119],"different":[121],"models":[124,202],"conduct":[126],"2,079":[131],"them,":[133],"focusing":[134],"correctness":[136],"informativeness.":[138],"Subsequently,":[139],"we":[140],"investigated":[141],"performance":[143],"evaluating":[149,204],"these":[150,156],"answers,":[151,169],"analyzing":[152],"consistency":[154],"between":[155],"evaluations.":[160],"find":[162],"style,":[165],"length":[166],"category":[172],"questions":[174],"bias":[176],"metrics.":[180,190],"However,":[181],"fine-grained":[182],"helps":[184],"mitigate":[185],"issue":[187],"some":[189],"Our":[191],"findings":[192],"have":[193],"important":[194],"implications":[195],"use":[198],"answering.":[207],"All":[208],"code":[209],"datasets":[211],"available":[213],"at":[214],"https://github.com/bugtig6351/lfqa_evaluation.":[215]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}