{"id":"https://openalex.org/W4401043061","doi":"https://doi.org/10.18653/v1/2024.findings-naacl.233","title":"Instruction-following Evaluation through Verbalizer Manipulation","display_name":"Instruction-following Evaluation through Verbalizer Manipulation","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4401043061","doi":"https://doi.org/10.18653/v1/2024.findings-naacl.233"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2024.findings-naacl.233","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-naacl.233","pdf_url":"https://aclanthology.org/2024.findings-naacl.233.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: NAACL 2024","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2024.findings-naacl.233.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100685011","display_name":"Shiyang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Shiyang Li","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100931149","display_name":"Jun Yan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jun Yan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106203239","display_name":"Hai Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hai Wang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101599872","display_name":"Zheng Tang","orcid":"https://orcid.org/0000-0002-3238-9263"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zheng Tang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009408707","display_name":"Xiang Ren","orcid":"https://orcid.org/0000-0001-8655-663X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiang Ren","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106102126","display_name":"Vijay Srinivasan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vijay Srinivasan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5044205212","display_name":"Hongxia Jin","orcid":"https://orcid.org/0009-0000-0222-4217"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hongxia Jin","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5100685011"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.6361,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.95954005,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"3678","last_page":"3692"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10826","display_name":"Behavioral and Psychological Studies","score":0.02449999935925007,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10826","display_name":"Behavioral and Psychological Studies","score":0.02449999935925007,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10789","display_name":"Interactive and Immersive Displays","score":0.022700000554323196,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.010400000028312206,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.623408854007721}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.623408854007721}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2024.findings-naacl.233","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-naacl.233","pdf_url":"https://aclanthology.org/2024.findings-naacl.233.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: NAACL 2024","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2024.findings-naacl.233","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-naacl.233","pdf_url":"https://aclanthology.org/2024.findings-naacl.233.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: NAACL 2024","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320332195","display_name":"Samsung","ror":"https://ror.org/04w3jy968"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4401043061.pdf","grobid_xml":"https://content.openalex.org/works/W4401043061.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"While":[0],"instruction-tuned":[1],"models":[2],"have":[3],"shown":[4],"remarkable":[5],"success":[6],"in":[7,39,50],"various":[8],"natural":[9,170],"language":[10],"processing":[11],"tasks,":[12],"accurately":[13,125],"evaluating":[14],"their":[15,166,196],"ability":[16,49,120],"to":[17,41,68,79,93,111,121,124,177,194],"follow":[18,126],"instructions":[19,27,43],"remains":[20],"challenging.Existing":[21],"benchmarks":[22],"primarily":[23],"focus":[24],"on":[25,116,168,183],"common":[26],"that":[28,151],"align":[29],"well":[30],"with":[31,73,76,107],"what":[32],"the":[33,66,70,113,127,152,172,184,189],"model":[34,67,77,136,175],"learned":[35],"during":[36],"training.However,":[37],"proficiency":[38],"responding":[40],"these":[42],"does":[44],"not":[45],"necessarily":[46],"imply":[47],"strong":[48],"instruction":[51],"following.In":[52],"this":[53],"paper,":[54],"we":[55],"propose":[56],"a":[57,130],"novel":[58],"instruction-following":[59,153,197],"evaluation":[60,132],"protocol":[61],"called":[62],"verbalizer":[63],"manipulation.It":[64],"instructs":[65],"verbalize":[69],"task":[71],"label":[72],"words":[74],"aligning":[75],"priors":[78,117],"different":[80,158],"extents,":[81],"adopting":[82],"verbalizers":[83,145],"from":[84],"highly":[85],"aligned":[86,95],"(e.g.,":[87,96],"outputting":[88,97],"\"positive\"":[89],"for":[90,99,146,191],"positive":[91,100],"sentiment),":[92],"minimally":[94],"\"negative\"":[98],"sentiment).Verbalizer":[101],"manipulation":[102],"can":[103],"be":[104],"seamlessly":[105],"integrated":[106],"any":[108],"classification":[109],"benchmark":[110],"examine":[112],"model's":[114],"reliance":[115],"and":[118,160],"its":[119],"override":[122],"them":[123],"instructions.We":[128],"conduct":[129],"comprehensive":[131],"of":[133,144,148,155],"four":[134],"major":[135],"families":[137,159],"across":[138,157],"nine":[139],"datasets,":[140],"employing":[141],"twelve":[142],"sets":[143],"each":[147],"them.We":[149],"observe":[150],"abilities":[154],"models,":[156],"scales,":[161],"are":[162],"significantly":[163],"distinguished":[164],"by":[165],"performance":[167],"less":[169],"verbalizers.Even":[171],"strongest":[173],"GPT-4":[174],"struggles":[176],"perform":[178],"better":[179],"than":[180],"random":[181],"guessing":[182],"most":[185],"challenging":[186],"verbalizer,":[187],"emphasizing":[188],"need":[190],"continued":[192],"advancements":[193],"improve":[195],"abilities.":[198]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2025-10-10T00:00:00"}