{"id":"https://openalex.org/W4403884725","doi":"https://doi.org/10.48550/arxiv.2410.03608","title":"TICKing All the Boxes: Generated Checklists Improve LLM Evaluation and Generation","display_name":"TICKing All the Boxes: Generated Checklists Improve LLM Evaluation and Generation","publication_year":2024,"publication_date":"2024-10-04","ids":{"openalex":"https://openalex.org/W4403884725","doi":"https://doi.org/10.48550/arxiv.2410.03608"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.03608","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.03608","pdf_url":"https://arxiv.org/pdf/2410.03608","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.03608","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Cook, Jonathan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Cook, Jonathan","raw_affiliation_strings":["University of Oxford"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oxford","institution_ids":["https://openalex.org/I40120149"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079315903","display_name":"Tim Rockt\u00e4schel","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rockt\u00e4schel, Tim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059094093","display_name":"Jakob Foerster","orcid":"https://orcid.org/0000-0001-9688-2498","observed_orcids":["https://orcid.org/0000-0001-9688-2498"]},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Foerster, Jakob","raw_affiliation_strings":["University of Oxford"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oxford","institution_ids":["https://openalex.org/I40120149"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039484875","display_name":"Dennis Aumiller","orcid":"https://orcid.org/0000-0001-8296-6797","observed_orcids":["https://orcid.org/0000-0001-8296-6797"]},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Aumiller, Dennis","raw_affiliation_strings":["Centre for AI , University College London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Centre for AI , University College London","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5114452841","display_name":"Alex Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Wang, Alex","raw_affiliation_strings":["Centre for AI , University College London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Centre for AI , University College London","institution_ids":["https://openalex.org/I45129253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5601000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5601000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.3167000114917755,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.016499999910593033,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8209999799728394},{"id":"https://openalex.org/keywords/automated-assessment","display_name":"automated assessment","score":0.593999981880188},{"id":"https://openalex.org/keywords/preference-judgments","display_name":"preference judgments","score":0.5329999923706055},{"id":"https://openalex.org/keywords/self-refinement","display_name":"self-refinement","score":0.7870000004768372},{"id":"https://openalex.org/keywords/inter-annotator-agreement","display_name":"inter-annotator agreement","score":0.9779999852180481}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.315762996673584}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.03608","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.03608","pdf_url":"https://arxiv.org/pdf/2410.03608","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.03608","doi":"https://doi.org/10.48550/arxiv.2410.03608","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.03608","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.03608","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.03608","pdf_url":"https://arxiv.org/pdf/2410.03608","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403884725.pdf","grobid_xml":"https://content.openalex.org/works/W4403884725.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7166831353","https://openalex.org/W4388718089","https://openalex.org/W4412944198","https://openalex.org/W4409363989","https://openalex.org/W4416036248","https://openalex.org/W4405714372","https://openalex.org/W4403345787","https://openalex.org/W7169548009","https://openalex.org/W4387596586","https://openalex.org/W4401208776"],"abstract_inverted_index":{"Given":[0],"the":[1,31,62,108,128,144,209],"widespread":[2],"adoption":[3],"and":[4,17,51,66,152,183],"usage":[5],"of":[6,20,64,113,127,146,197,216],"Large":[7],"Language":[8],"Models":[9],"(LLMs),":[10],"it":[11],"is":[12,49,221],"crucial":[13],"to":[14,57,136,157,174,193,223,228,238,247],"have":[15,29],"flexible":[16],"interpretable":[18,81],"evaluations":[19,86],"their":[21],"instruction-following":[22],"ability.":[23],"Preference":[24],"judgments":[25],"between":[26,149],"model":[27],"outputs":[28],"become":[30],"de":[32],"facto":[33],"evaluation":[34,82,104],"standard,":[35],"despite":[36],"distilling":[37],"complex,":[38],"multi-faceted":[39,219],"preferences":[40],"into":[41,110],"a":[42,78,111,120,124,137,225],"single":[43],"ranking.":[44],"Furthermore,":[45],"as":[46,155],"human":[47,153,239],"annotation":[48],"slow":[50],"costly,":[52],"LLMs":[53,98],"are":[54],"increasingly":[55],"used":[56,173],"make":[58],"these":[59],"judgments,":[60],"at":[61],"expense":[63],"reliability":[65],"interpretability.":[67],"In":[68,214],"this":[69],"work,":[70],"we":[71,250],"propose":[72],"TICK":[73,134],"(Targeted":[74],"Instruct-evaluation":[75],"with":[76,87,202,242],"ChecKlists),":[77],"fully":[79],"automated,":[80],"protocol":[83],"that":[84,106,132,168],"structures":[85],"LLM-generated,":[88],"instruction-specific":[89],"checklists.":[90],"We":[91,130,165],"first":[92],"show":[93,167],"that,":[94],"given":[95],"an":[96,159,163,194],"instruction,":[97],"can":[99,171],"reliably":[100],"produce":[101],"high-quality,":[102],"tailored":[103],"checklists":[105,237],"decompose":[107],"instruction":[109,211],"series":[112],"YES/NO":[114],"questions.":[115],"Each":[116],"question":[117],"asks":[118],"whether":[119],"candidate":[121],"response":[122],"meets":[123],"specific":[125],"requirement":[126],"instruction.":[129],"demonstrate":[131],"using":[133],"leads":[135,192],"significant":[138],"increase":[139,252],"(46.4%":[140],"$\\to$":[141,256],"52.2%)":[142],"in":[143],"frequency":[145],"exact":[147],"agreements":[148],"LLM":[150,160,231,245],"judgements":[151],"preferences,":[154],"compared":[156],"having":[158],"directly":[161,243],"score":[162],"output.":[164],"then":[166],"STICK":[169,186,203],"(Self-TICK)":[170],"be":[172,224],"improve":[175],"generation":[176],"quality":[177],"across":[178],"multiple":[179],"benchmarks":[180],"via":[181],"self-refinement":[182,187],"Best-of-N":[184,200],"selection.":[185],"on":[188,208],"LiveBench":[189],"reasoning":[190],"tasks":[191],"absolute":[195,206],"gain":[196],"$+$7.8%,":[198],"whilst":[199],"selection":[201],"attains":[204],"$+$6.3%":[205],"improvement":[207],"real-world":[210],"dataset,":[212],"WildBench.":[213],"light":[215],"this,":[217],"structured,":[218],"self-improvement":[220],"shown":[222],"promising":[226],"way":[227],"further":[229],"advance":[230],"capabilities.":[232],"Finally,":[233],"by":[234],"providing":[235],"LLM-generated":[236],"evaluators":[240],"tasked":[241],"scoring":[244],"responses":[246],"WildBench":[248],"instructions,":[249],"notably":[251],"inter-annotator":[253],"agreement":[254],"(0.194":[255],"0.256).":[257]},"counts_by_year":[],"updated_date":"2026-09-30T09:27:27.532063","created_date":"2024-10-30T00:00:00"}