{"id":"https://openalex.org/W4416034825","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.622","title":"Benchmarking for Domain-Specific LLMs: A Case Study on Academia and Beyond","display_name":"Benchmarking for Domain-Specific LLMs: A Case Study on Academia and Beyond","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416034825","doi":"https://doi.org/10.18653/v1/2025.findings-emnlp.622"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.622","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.622","pdf_url":"https://aclanthology.org/2025.findings-emnlp.622.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-emnlp.622.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111976364","display_name":"Rubing Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rubing Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101880563","display_name":"Jiaxin Wu","orcid":"https://orcid.org/0000-0002-2015-8206"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiaxin Wu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100370396","display_name":"Jian Wang","orcid":"https://orcid.org/0000-0002-0579-3041"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jian Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089910828","display_name":"Xu-Lu Zhang","orcid":"https://orcid.org/0000-0003-2473-460X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xulu Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043696243","display_name":"Wenqi Fan","orcid":"https://orcid.org/0000-0002-4049-1233"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wenqi Fan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024599321","display_name":"Chenghua Lin","orcid":"https://orcid.org/0000-0003-3454-2468"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chenghua Lin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034340826","display_name":"Xiaoyong Wei","orcid":"https://orcid.org/0000-0001-9677-4227"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiaoyong Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100404102","display_name":"Qing Li","orcid":"https://orcid.org/0000-0002-4982-1024"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li Qing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.1788,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.8745203,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"11606","last_page":"11619"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12171","display_name":"Open Education and E-Learning","score":0.08749999850988388,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12171","display_name":"Open Education and E-Learning","score":0.08749999850988388,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13034","display_name":"Information Systems Education and Curriculum Development","score":0.05050000175833702,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11937","display_name":"Research Data Management Practices","score":0.047200001776218414,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.6366999745368958},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.3402000069618225},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.29249998927116394},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.2542000114917755}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.6366999745368958},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3804999887943268},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.3571999967098236},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.3402000069618225},{"id":"https://openalex.org/C110354214","wikidata":"https://www.wikidata.org/wiki/Q6314146","display_name":"Engineering management","level":1,"score":0.30239999294281006},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.296999990940094},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.29249998927116394},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.2797999978065491},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2542000114917755},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.2533999979496002}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-emnlp.622","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.622","pdf_url":"https://aclanthology.org/2025.findings-emnlp.622.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-emnlp.622","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-emnlp.622","pdf_url":"https://aclanthology.org/2025.findings-emnlp.622.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4857298105","display_name":null,"funder_award_id":"62372314","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322598","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416034825.pdf","grobid_xml":"https://content.openalex.org/works/W4416034825.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"increasing":[1],"demand":[2],"for":[3,74],"domain-specific":[4,55,75],"evaluation":[5],"of":[6,16,25,43,54,89,101,116,133,157],"large":[7,30],"language":[8],"models":[9],"(LLMs)":[10],"has":[11],"led":[12],"to":[13,22,37,153],"the":[14,23,41,50,71,87,98,102,114,131,145],"development":[15],"numerous":[17],"benchmarks.These":[18],"efforts":[19],"often":[20],"adhere":[21],"principle":[24,73,88],"data":[26,66],"scaling,":[27],"relying":[28],"on":[29,49,143],"corpora":[31],"or":[32],"extensive":[33],"question-answer":[34],"(QA)":[35],"sets":[36],"ensure":[38],"broad":[39],"coverage.However,":[40],"impact":[42],"corpus":[44],"and":[45,52,91,111,150],"QA":[46],"set":[47],"design":[48],"precision":[51,107],"recall":[53,95],"LLM":[56],"performance":[57],"remains":[58],"poorly":[59],"understood.In":[60],"this":[61,140],"paper,":[62],"we":[63,78,119],"argue":[64],"that":[65],"scaling":[67],"is":[68,148],"not":[69],"always":[70],"optimal":[72],"benchmark":[76],"construction.Instead,":[77],"introduce":[79],"COMP-COMP,":[80],"an":[81],"iterative":[82],"benchmarking":[83],"framework":[84,147],"grounded":[85],"in":[86,130],"comprehensiveness":[90],"compactness.Comprehensiveness":[92],"ensures":[93],"semantic":[94],"by":[96,108],"covering":[97],"full":[99],"breadth":[100],"domain,":[103],"while":[104],"compactness":[105],"improves":[106],"reducing":[109],"redundancy":[110],"noise.To":[112],"demonstrate":[113],"effectiveness":[115],"our":[117],"approach,":[118],"present":[120],"a":[121,126,135,154],"case":[122],"study":[123,141],"conducted":[124],"at":[125],"wellrenowned":[127],"university,":[128],"resulting":[129],"creation":[132],"PolyBench,":[134],"large-scale,":[136],"high-quality":[137],"academic":[138],"benchmark.Although":[139],"focuses":[142],"academia,":[144],"COMP-COMP":[146],"domain-agnostic":[149],"readily":[151],"adaptable":[152],"wide":[155],"range":[156],"specialized":[158],"fields.":[159]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-06-16T09:24:06.705377","created_date":"2025-11-08T00:00:00"}