{"id":"https://openalex.org/W4411241405","doi":"https://doi.org/10.1145/3744663","title":"Evaluation of Question Answering Systems: Complexity of Judging a Natural Language","display_name":"Evaluation of Question Answering Systems: Complexity of Judging a Natural Language","publication_year":2025,"publication_date":"2025-06-12","ids":{"openalex":"https://openalex.org/W4411241405","doi":"https://doi.org/10.1145/3744663"},"language":"en","primary_location":{"id":"doi:10.1145/3744663","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3744663","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3744663","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111306296","display_name":"Amer Farea","orcid":null},"institutions":[{"id":"https://openalex.org/I166825849","display_name":"Tampere University","ror":"https://ror.org/033003e23","country_code":"FI","type":"education","lineage":["https://openalex.org/I166825849"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Amer Farea","raw_affiliation_strings":["Information Technology and Communication Sciences, Tampere University","Information Technology, Tampere University, Tampere, Finland"],"raw_orcid":"https://orcid.org/0000-0001-7145-7137","affiliations":[{"raw_affiliation_string":"Information Technology and Communication Sciences, Tampere University","institution_ids":["https://openalex.org/I166825849"]},{"raw_affiliation_string":"Information Technology, Tampere University, Tampere, Finland","institution_ids":["https://openalex.org/I166825849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103272144","display_name":"Zhen Yang","orcid":"https://orcid.org/0000-0001-6258-4783"},"institutions":[{"id":"https://openalex.org/I166825849","display_name":"Tampere University","ror":"https://ror.org/033003e23","country_code":"FI","type":"education","lineage":["https://openalex.org/I166825849"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Zhen Yang","raw_affiliation_strings":["Information Technology and Communication Sciences, Tampere University","Information Technology, Tampere University, Tampere Finland"],"raw_orcid":"https://orcid.org/0000-0001-6258-4783","affiliations":[{"raw_affiliation_string":"Information Technology and Communication Sciences, Tampere University","institution_ids":["https://openalex.org/I166825849"]},{"raw_affiliation_string":"Information Technology, Tampere University, Tampere Finland","institution_ids":["https://openalex.org/I166825849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064309868","display_name":"Kien Duong","orcid":null},"institutions":[{"id":"https://openalex.org/I166825849","display_name":"Tampere University","ror":"https://ror.org/033003e23","country_code":"FI","type":"education","lineage":["https://openalex.org/I166825849"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Kien Duong","raw_affiliation_strings":["Information Technology and Communication Sciences, Tampere University","Information Technology, Tampere University, Tampere Finland"],"raw_orcid":"https://orcid.org/0009-0009-3508-0972","affiliations":[{"raw_affiliation_string":"Information Technology and Communication Sciences, Tampere University","institution_ids":["https://openalex.org/I166825849"]},{"raw_affiliation_string":"Information Technology, Tampere University, Tampere Finland","institution_ids":["https://openalex.org/I166825849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005343110","display_name":"Nadeesha Perera","orcid":"https://orcid.org/0000-0002-9907-5939"},"institutions":[{"id":"https://openalex.org/I166825849","display_name":"Tampere University","ror":"https://ror.org/033003e23","country_code":"FI","type":"education","lineage":["https://openalex.org/I166825849"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Nadeesha Perera","raw_affiliation_strings":["Information Technology and Communication Sciences, Tampere University","Information Technology, Tampere University, Tampere Finland"],"raw_orcid":"https://orcid.org/0000-0002-9907-5939","affiliations":[{"raw_affiliation_string":"Information Technology and Communication Sciences, Tampere University","institution_ids":["https://openalex.org/I166825849"]},{"raw_affiliation_string":"Information Technology, Tampere University, Tampere Finland","institution_ids":["https://openalex.org/I166825849"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057625030","display_name":"Frank Emmert\u2010Streib","orcid":"https://orcid.org/0000-0003-0745-5641"},"institutions":[{"id":"https://openalex.org/I166825849","display_name":"Tampere University","ror":"https://ror.org/033003e23","country_code":"FI","type":"education","lineage":["https://openalex.org/I166825849"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Frank Emmert-Streib","raw_affiliation_strings":["Information Technology and Communication Sciences, Tampere University","Information Technology, Tampere University, Tampere Finland"],"raw_orcid":"https://orcid.org/0000-0003-0745-5641","affiliations":[{"raw_affiliation_string":"Information Technology and Communication Sciences, Tampere University","institution_ids":["https://openalex.org/I166825849"]},{"raw_affiliation_string":"Information Technology, Tampere University, Tampere Finland","institution_ids":["https://openalex.org/I166825849"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I166825849"],"apc_list":null,"apc_paid":null,"fwci":6.3222,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.96060635,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"58","issue":"1","first_page":"1","last_page":"43"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.9020726680755615},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.8005601167678833},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.5733245611190796},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.4982888698577881},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.49018484354019165},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45057398080825806},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.36587876081466675},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.32755863666534424}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.9020726680755615},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.8005601167678833},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5733245611190796},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.4982888698577881},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.49018484354019165},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45057398080825806},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.36587876081466675},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.32755863666534424},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3744663","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3744663","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},{"id":"pmh:oai:trepo.tuni.fi:10024/232818","is_oa":true,"landing_page_url":"https://trepo.tuni.fi/handle/10024/232818","pdf_url":null,"source":{"id":"https://openalex.org/S7407055260","display_name":"Trepo - Institutional Repository of Tampere University","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"reviewarticle"}],"best_oa_location":{"id":"doi:10.1145/3744663","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3744663","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":105,"referenced_works":["https://openalex.org/W160318044","https://openalex.org/W1552847225","https://openalex.org/W1593271688","https://openalex.org/W1607035479","https://openalex.org/W1785350809","https://openalex.org/W1895577753","https://openalex.org/W1956340063","https://openalex.org/W1973674431","https://openalex.org/W1979263599","https://openalex.org/W1982897610","https://openalex.org/W1985610876","https://openalex.org/W2036001619","https://openalex.org/W2037960075","https://openalex.org/W2047382442","https://openalex.org/W2069143585","https://openalex.org/W2125436846","https://openalex.org/W2140054881","https://openalex.org/W2145618437","https://openalex.org/W2150824314","https://openalex.org/W2156644463","https://openalex.org/W2170872814","https://openalex.org/W2183366530","https://openalex.org/W2236262502","https://openalex.org/W2250342921","https://openalex.org/W2250875036","https://openalex.org/W2251692155","https://openalex.org/W2252136820","https://openalex.org/W2561368124","https://openalex.org/W2575842049","https://openalex.org/W2584220694","https://openalex.org/W2594829673","https://openalex.org/W2600463316","https://openalex.org/W2606333299","https://openalex.org/W2612560781","https://openalex.org/W2618735189","https://openalex.org/W2755637027","https://openalex.org/W2756675635","https://openalex.org/W2759200442","https://openalex.org/W2766317792","https://openalex.org/W2781827132","https://openalex.org/W2804292122","https://openalex.org/W2810503719","https://openalex.org/W2887097088","https://openalex.org/W2888302696","https://openalex.org/W2891416139","https://openalex.org/W2897173776","https://openalex.org/W2899008850","https://openalex.org/W2903314293","https://openalex.org/W2903376039","https://openalex.org/W2905377481","https://openalex.org/W2912102502","https://openalex.org/W2912924812","https://openalex.org/W2913352150","https://openalex.org/W2941573484","https://openalex.org/W2944069152","https://openalex.org/W2947771965","https://openalex.org/W2949021450","https://openalex.org/W2955202831","https://openalex.org/W2963527228","https://openalex.org/W2963672599","https://openalex.org/W2963918774","https://openalex.org/W2963963993","https://openalex.org/W2964138343","https://openalex.org/W2964223283","https://openalex.org/W2966087730","https://openalex.org/W2970796366","https://openalex.org/W2972551450","https://openalex.org/W2980346149","https://openalex.org/W2980401255","https://openalex.org/W2983309655","https://openalex.org/W2992007572","https://openalex.org/W2992347006","https://openalex.org/W3006781240","https://openalex.org/W3011309387","https://openalex.org/W3015848964","https://openalex.org/W3022187094","https://openalex.org/W3034273250","https://openalex.org/W3034630799","https://openalex.org/W3034879633","https://openalex.org/W3035656139","https://openalex.org/W3044495139","https://openalex.org/W3087927280","https://openalex.org/W3095319910","https://openalex.org/W3102659883","https://openalex.org/W3104405162","https://openalex.org/W3113362544","https://openalex.org/W3153094109","https://openalex.org/W3156487623","https://openalex.org/W3165304031","https://openalex.org/W3169841173","https://openalex.org/W3178077565","https://openalex.org/W3196385594","https://openalex.org/W3211339323","https://openalex.org/W4200454073","https://openalex.org/W4205458180","https://openalex.org/W4206256378","https://openalex.org/W4206958985","https://openalex.org/W4213052788","https://openalex.org/W4288269198","https://openalex.org/W4296557505","https://openalex.org/W4364320763","https://openalex.org/W4389519239","https://openalex.org/W4399020384","https://openalex.org/W4401042286","https://openalex.org/W4402580364"],"related_works":["https://openalex.org/W3157284875","https://openalex.org/W2147241511","https://openalex.org/W2259406085","https://openalex.org/W2099715052","https://openalex.org/W4226247999","https://openalex.org/W3090872036","https://openalex.org/W3209772662","https://openalex.org/W4200629926","https://openalex.org/W4220955952","https://openalex.org/W4287868219"],"abstract_inverted_index":{"Question":[0],"answering":[1],"(QA)":[2],"systems":[3,48,62],"are":[4,72,165],"a":[5,98,106,151],"leading":[6],"and":[7,31,104,116,146,157],"rapidly":[8],"advancing":[9],"field":[10],"of":[11,18,60,67,101,123,143,154],"natural":[12,27],"language":[13],"processing":[14],"(NLP)":[15],"research.":[16],"One":[17],"their":[19,93,155],"key":[20],"advantages":[21],"is":[22,63],"that":[23,164],"they":[24],"enable":[25],"more":[26,136],"interactions":[28],"between":[29],"humans":[30],"machines,":[32],"such":[33],"as":[34,65],"in":[35],"virtual":[36],"assistants":[37],"or":[38],"search":[39],"engines.":[40],"Over":[41],"the":[42,58,68,141],"past":[43],"few":[44],"decades,":[45],"many":[46,66,122],"QA":[47,55,144,173],"have":[49,133],"been":[50,134],"developed":[51],"to":[52,83,92,149],"handle":[53],"diverse":[54],"tasks.":[56,174],"However,":[57],"evaluation":[59,70,102],"these":[61,124],"intricate,":[64],"available":[69],"scores":[71,103,125],"not":[73],"task-agnostic.":[74],"Furthermore,":[75],"translating":[76],"human":[77],"judgment":[78],"into":[79],"measurable":[80],"metrics":[81],"continues":[82],"be":[84],"an":[85],"open":[86],"issue.":[87],"These":[88],"complexities":[89],"add":[90],"challenges":[91],"assessment.":[94],"This":[95],"survey":[96],"provides":[97],"systematic":[99,169],"overview":[100],"introduces":[105],"taxonomy":[107],"with":[108],"two":[109],"main":[110],"branches:":[111],"Human-Centric":[112],"Evaluation":[113,118],"Scores":[114,119],"(HCES)":[115],"Automatic":[117],"(AES).":[120],"Since":[121],"were":[126],"originally":[127],"designed":[128],"for":[129,167],"specific":[130],"tasks":[131],"but":[132],"applied":[135],"generally,":[137],"we":[138,160],"also":[139],"cover":[140],"basics":[142],"frameworks":[145],"core":[147],"paradigms":[148],"provide":[150],"deeper":[152],"understanding":[153],"capabilities":[156],"limitations.":[158],"Lastly,":[159],"discuss":[161],"benchmark":[162],"datasets":[163],"critical":[166],"conducting":[168],"evaluations":[170],"across":[171],"various":[172]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3}],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2025-10-10T00:00:00"}