{"id":"https://openalex.org/W7166885666","doi":"https://doi.org/10.18653/v1/2026.acl-short.58","title":"Revisiting Evaluation of Question Answering Systems in Low-Resource Indic Languages: Bridging Human and Metric Alignment","display_name":"Revisiting Evaluation of Question Answering Systems in Low-Resource Indic Languages: Bridging Human and Metric Alignment","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166885666","doi":"https://doi.org/10.18653/v1/2026.acl-short.58"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.acl-short.58","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-short.58","pdf_url":"https://aclanthology.org/2026.acl-short.58.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.acl-short.58.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139776195","display_name":"Anuj Kumar","orcid":null},"institutions":[{"id":"https://openalex.org/I4210127441","display_name":"Indian Institute of Technology Jammu","ror":"https://ror.org/02f0vsw63","country_code":"IN","type":"education","lineage":["https://openalex.org/I4210127441"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Anuj Kumar","raw_affiliation_strings":["Department of Computer Science & Engineering , Indian Institute of Technology Jammu , India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science & Engineering , Indian Institute of Technology Jammu , India","institution_ids":["https://openalex.org/I4210127441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129641810","display_name":"Satyadev Ahlawat","orcid":null},"institutions":[{"id":"https://openalex.org/I4210127441","display_name":"Indian Institute of Technology Jammu","ror":"https://ror.org/02f0vsw63","country_code":"IN","type":"education","lineage":["https://openalex.org/I4210127441"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Satyadev Ahlawat","raw_affiliation_strings":["Department of Electrical Engineering , Indian Institute of Technology Jammu , India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering , Indian Institute of Technology Jammu , India","institution_ids":["https://openalex.org/I4210127441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060469822","display_name":"Yamuna Prasad","orcid":"https://orcid.org/0000-0002-3709-7956"},"institutions":[{"id":"https://openalex.org/I4210127441","display_name":"Indian Institute of Technology Jammu","ror":"https://ror.org/02f0vsw63","country_code":"IN","type":"education","lineage":["https://openalex.org/I4210127441"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Yamuna Prasad","raw_affiliation_strings":["Department of Computer Science & Engineering , Indian Institute of Technology Jammu , India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science & Engineering , Indian Institute of Technology Jammu , India","institution_ids":["https://openalex.org/I4210127441"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5139848250","display_name":"Virendra Singh","orcid":null},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Virendra Singh","raw_affiliation_strings":["Department of Electrical Engineering , Indian Institute of Technology Bombay , India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering , Indian Institute of Technology Bombay , India","institution_ids":["https://openalex.org/I162827531"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.84431076,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"703","last_page":"718"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.40880000591278076,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.40880000591278076,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.15610000491142273,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.07280000299215317,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bridging","display_name":"Bridging (networking)","score":0.6942999958992004},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.670799970626831},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.4253000020980835},{"id":"https://openalex.org/keywords/information-system","display_name":"Information system","score":0.27570000290870667}],"concepts":[{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.6942999958992004},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.670799970626831},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6078000068664551},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4657999873161316},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.4253000020980835},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4090999960899353},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3269999921321869},{"id":"https://openalex.org/C180198813","wikidata":"https://www.wikidata.org/wiki/Q121182","display_name":"Information system","level":2,"score":0.27570000290870667},{"id":"https://openalex.org/C193429382","wikidata":"https://www.wikidata.org/wiki/Q232405","display_name":"Metric system","level":2,"score":0.260699987411499},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.25200000405311584}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.acl-short.58","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-short.58","pdf_url":"https://aclanthology.org/2026.acl-short.58.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.acl-short.58","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.acl-short.58","pdf_url":"https://aclanthology.org/2026.acl-short.58.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166885666.pdf","grobid_xml":"https://content.openalex.org/works/W7166885666.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Evaluating":[0],"Question":[1,105],"Answering":[2,106],"(QA)":[3],"systems":[4],"in":[5,58,68,192],"low-resource":[6,69,193],"Indic":[7,194],"languages":[8,29],"remains":[9],"challenging":[10],"due":[11],"to":[12,36,65],"the":[13,22,39,96],"scarcity":[14],"of":[15,24,41],"annotated":[16],"data,":[17],"high":[18],"linguistic":[19],"diversity,":[20],"and":[21,54,61,84,118,141,160,177],"absence":[23],"reliable":[25,186],"evaluation":[26,109],"metrics.Many":[27],"Indian":[28],"are":[30],"severely":[31],"underrepresented,":[32],"making":[33],"it":[34],"difficult":[35],"accurately":[37],"assess":[38],"performance":[40],"Large":[42],"Language":[43],"Models":[44],"(LLMs)":[45],"on":[46,136],"QA":[47,70,128,191],"tasks.Commonly":[48],"used":[49],"metrics":[50,72],"like":[51],"BLEU,":[52],"ROUGE-L,":[53],"BERTScore,":[55],"while":[56],"successful":[57],"machine":[59],"translation":[60],"resource-rich":[62],"scenarios,":[63],"tend":[64],"perform":[66],"poorly":[67],"settings.These":[71],"often":[73],"exhibit":[74],"issues":[75],"such":[76],"as":[77,133,135,155],"compressed":[78],"scoring":[79],"ranges,":[80],"excessive":[81],"zero":[82],"scores,":[83],"weak":[85],"alignment":[86],"with":[87,152,181],"human":[88,153,182],"judgments.To":[89],"overcome":[90],"these":[91],"limitations,":[92],"this":[93],"work":[94],"introduces":[95],"LRM":[97,146,167],"2":[98,147,168],"QAS":[99,148,169],"(Language":[100],"Robust":[101],"Multi-aspect":[102],"Metrics":[103],"for":[104,188],"Systems).This":[107],"composite":[108],"framework":[110,187],"integrates":[111],"semantic":[112],"similarity,":[113],"factual":[114],"completeness,":[115],"numerical":[116],"accuracy,":[117],"contextual":[119],"relevance.The":[120],"proposed":[121],"metric":[122],"is":[123],"evaluated":[124],"across":[125],"eight":[126],"Indic-language":[127],"tasks":[129],"using":[130],"multiple":[131],"LLMs,":[132],"well":[134],"open-domain":[137],"benchmarks":[138],"NaturalQuestions":[139],"(NQ)":[140],"Triv-iaQA":[142],"(TQ).Across":[143],"all":[144],"settings,":[145],"demonstrates":[149],"stronger":[150],"agreement":[151],"evaluation,":[154],"measured":[156],"by":[157],"Pearson,":[158],"Spearman,":[159],"Kendall":[161],"correlation":[162],"coefficients.Experimental":[163],"findings":[164],"highlight":[165],"that":[166],"provides":[170],"more":[171,179],"precise":[172],"distinctions":[173],"between":[174],"model":[175],"outputs":[176],"aligns":[178],"closely":[180],"judgment,":[183],"offering":[184],"a":[185],"evaluating":[189],"multilingual":[190],"languages.":[195]},"counts_by_year":[],"updated_date":"2026-08-24T07:32:12.397491","created_date":"2026-07-02T00:00:00"}