{"id":"https://openalex.org/W4406059439","doi":"https://doi.org/10.48550/arxiv.2407.13522","title":"INDIC QA BENCHMARK: A Multilingual Benchmark to Evaluate Question Answering capability of LLMs for Indic Languages","display_name":"INDIC QA BENCHMARK: A Multilingual Benchmark to Evaluate Question Answering capability of LLMs for Indic Languages","publication_year":2024,"publication_date":"2024-07-18","ids":{"openalex":"https://openalex.org/W4406059439","doi":"https://doi.org/10.48550/arxiv.2407.13522"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2407.13522","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.13522","pdf_url":"https://arxiv.org/pdf/2407.13522","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2407.13522","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087279328","display_name":"Abhishek Kumar Singh","orcid":"https://orcid.org/0000-0002-1327-8533"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Singh, Abhishek Kumar","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102707912","display_name":"Rudra Murthy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"kumar, Vishwajeet","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100746738","display_name":"Vishwajeet Kumar","orcid":"https://orcid.org/0000-0002-5783-5207"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Murthy, Rudra","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102383724","display_name":"Jaydeep Sen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sen, Jaydeep","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089606464","display_name":"Ganesh Ramakrishnan","orcid":"https://orcid.org/0000-0003-4533-2490"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mittal, Ashish","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Ramakrishnan, Ganesh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ramakrishnan, Ganesh","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5087279328"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9703999757766724,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9545000195503235,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8868997693061829},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.5382354855537415},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5180099606513977},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4800663888454437},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.4014352560043335},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3579961657524109},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.24896082282066345},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.08789917826652527},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.07082027196884155}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8868997693061829},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.5382354855537415},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5180099606513977},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4800663888454437},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.4014352560043335},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3579961657524109},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.24896082282066345},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.08789917826652527},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.07082027196884155}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2407.13522","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.13522","pdf_url":"https://arxiv.org/pdf/2407.13522","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2407.13522","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2407.13522","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2407.13522","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2407.13522","pdf_url":"https://arxiv.org/pdf/2407.13522","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4406059439.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2384605597","https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W2387743295","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W4246352526","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"perform":[4],"well":[5],"on":[6],"unseen":[7],"tasks":[8],"in":[9,14,46,68,79,115,134],"English,":[10],"but":[11],"their":[12,80],"abilities":[13],"non":[15],"English":[16,76,94],"languages":[17,71],"are":[18,91,100],"less":[19],"explored":[20],"due":[21,72],"to":[22,73,93,125,144],"limited":[23],"benchmarks":[24],"and":[25,54,97,148],"training":[26,81],"data.":[27,82],"To":[28],"bridge":[29],"this":[30],"gap,":[31],"we":[32,123],"introduce":[33],"the":[34,86,98,104],"Indic":[35,121],"QA":[36],"Benchmark,":[37],"a":[38,74,141],"large":[39],"dataset":[40],"for":[41,95,107],"context":[42],"grounded":[43],"question":[44,131],"answering":[45,132],"11":[47],"major":[48],"Indian":[49],"languages,":[50],"covering":[51],"both":[52],"extractive":[53],"abstractive":[55],"tasks.":[56],"Evaluations":[57],"of":[58],"multilingual":[59,112,150],"LLMs,":[60,113],"including":[61],"instruction":[62],"finetuned":[63],"versions,":[64],"revealed":[65],"weak":[66],"performance":[67],"low":[69,116,135],"resource":[70,117,136,143],"strong":[75],"language":[77,106],"bias":[78],"We":[83],"also":[84],"investigated":[85],"Translate":[87],"Test":[88],"paradigm,where":[89],"inputs":[90],"translated":[92,101],"processing":[96],"results":[99],"back":[102],"into":[103,129],"source":[105],"output.":[108],"This":[109,138],"approach":[110],"outperformed":[111],"particularly":[114],"settings.":[118],"By":[119],"releasing":[120],"QA,":[122],"aim":[124],"promote":[126],"further":[127],"research":[128],"LLMs":[130],"capabilities":[133],"languages.":[137],"benchmark":[139],"offers":[140],"critical":[142],"address":[145],"existing":[146],"limitations":[147],"foster":[149],"understanding.":[151]},"counts_by_year":[],"updated_date":"2026-03-12T08:34:05.389933","created_date":"2025-10-10T00:00:00"}