{"id":"https://openalex.org/W4384648604","doi":"https://doi.org/10.48550/arxiv.2307.08689","title":"COLLIE: Systematic Construction of Constrained Text Generation Tasks","display_name":"COLLIE: Systematic Construction of Constrained Text Generation Tasks","publication_year":2023,"publication_date":"2023-07-17","ids":{"openalex":"https://openalex.org/W4384648604","doi":"https://doi.org/10.48550/arxiv.2307.08689"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2307.08689","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.08689","pdf_url":"https://arxiv.org/pdf/2307.08689","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2307.08689","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Yao, Shunyu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yao, Shunyu","raw_affiliation_strings":["Department of Computer Science , Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101444574","display_name":"Howard Chen","orcid":"https://orcid.org/0000-0001-8628-7847","observed_orcids":["https://orcid.org/0000-0001-8628-7847"]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chen, Howard","raw_affiliation_strings":["Department of Computer Science , Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056227896","display_name":"Austin W. Hanjie","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hanjie, Austin W.","raw_affiliation_strings":["Department of Computer Science , Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032223753","display_name":"Runzhe Yang","orcid":"https://orcid.org/0000-0001-8891-5149","observed_orcids":["https://orcid.org/0000-0001-8891-5149","https://orcid.org/0009-0002-5452-5681"]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang, Runzhe","raw_affiliation_strings":["Department of Computer Science , Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , Princeton University","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025205227","display_name":"Karthik Narasimhan","orcid":"https://orcid.org/0000-0001-9894-9983","observed_orcids":["https://orcid.org/0000-0001-9894-9983"]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Narasimhan, Karthik","raw_affiliation_strings":["Department of Computer Science , Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science , Princeton University","institution_ids":["https://openalex.org/I20089843"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I20089843"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7954000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7954000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.18320000171661377,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.009700000286102295,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/compositional-constraints","display_name":"compositional constraints","score":0.9459999799728394},{"id":"https://openalex.org/keywords/instruction-tuned-language-models","display_name":"instruction-tuned language models","score":0.8629999756813049},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6570000052452087},{"id":"https://openalex.org/keywords/logical-reasoning","display_name":"logical reasoning","score":0.4269999861717224},{"id":"https://openalex.org/keywords/semantic-planning","display_name":"semantic planning","score":0.8629999756813049},{"id":"https://openalex.org/keywords/counting-task","display_name":"counting task","score":0.4440000057220459},{"id":"https://openalex.org/keywords/language-comprehension","display_name":"language comprehension","score":0.5740000009536743}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8230510950088501},{"id":"https://openalex.org/C2777206241","wikidata":"https://www.wikidata.org/wiki/Q194431","display_name":"Paragraph","level":2,"score":0.6725094318389893},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.6230035424232483},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.6160498857498169},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6119129657745361},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5656811594963074},{"id":"https://openalex.org/C26022165","wikidata":"https://www.wikidata.org/wiki/Q8091","display_name":"Grammar","level":2,"score":0.4983201026916504},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4716196358203888},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.46047914028167725},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4246901869773865},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.33755531907081604},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.15083494782447815},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2307.08689","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.08689","pdf_url":"https://arxiv.org/pdf/2307.08689","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2307.08689","doi":"https://doi.org/10.48550/arxiv.2307.08689","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2307.08689","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2307.08689","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.08689","pdf_url":"https://arxiv.org/pdf/2307.08689","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"id":"https://metadata.un.org/sdg/4","score":0.8199999928474426,"display_name":"Quality education"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G4113729616","display_name":"RI: Medium: Improving grounding, generalization and contextual reasoning in vision and language models","funder_award_id":"2107048","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4384648604.pdf","grobid_xml":"https://content.openalex.org/works/W4384648604.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4312091623","https://openalex.org/W4367369699","https://openalex.org/W4282921036","https://openalex.org/W4387963756","https://openalex.org/W4396651235","https://openalex.org/W4382618497","https://openalex.org/W3021914396","https://openalex.org/W4400025287","https://openalex.org/W3137121515","https://openalex.org/W4386942562"],"abstract_inverted_index":{"Text":[0],"generation":[1,27,67],"under":[2],"constraints":[3,64,153],"have":[4,41],"seen":[5],"increasing":[6],"interests":[7],"in":[8,156],"natural":[9],"language":[10,20,124],"processing,":[11],"especially":[12],"with":[13,65,109],"the":[14,59,106,144,157],"rapidly":[15],"improving":[16],"capabilities":[17],"of":[18,61,90],"large":[19],"models.":[21],"However,":[22],"existing":[23],"benchmarks":[24],"for":[25,46,87],"constrained":[26],"usually":[28],"focus":[29],"on":[30],"fixed":[31],"constraint":[32,95,114],"types":[33],"(e.g.,generate":[34],"a":[35,54,94,98],"sentence":[36],"containing":[37],"certain":[38],"words)":[39],"that":[40,57],"proved":[42],"to":[43,130,136,149],"be":[44,137],"easy":[45],"state-of-the-art":[47,122],"models":[48,125],"like":[49],"GPT-4.":[50],"We":[51,83,116],"present":[52],"COLLIE,":[53,103],"grammar-based":[55],"framework":[56],"allows":[58],"specification":[60],"rich,":[62],"compositional":[63],"diverse":[66],"levels":[68],"(word,":[69],"sentence,":[70],"paragraph,":[71],"passage)":[72],"and":[73,97,126,139,141,154],"modeling":[74],"challenges":[75],"(e.g.,language":[76],"understanding,":[77],"logical":[78],"reasoning,":[79],"counting,":[80],"semantic":[81],"planning).":[82],"also":[84],"develop":[85,150],"tools":[86],"automatic":[88],"extraction":[89],"task":[91],"instances":[92,111],"given":[93],"structure":[96],"raw":[99],"text":[100],"corpus.":[101],"Using":[102],"we":[104,142],"compile":[105],"COLLIE-v1":[107],"dataset":[108],"2080":[110],"comprising":[112],"13":[113],"structures.":[115],"perform":[117],"systematic":[118],"experiments":[119],"across":[120],"five":[121],"instruction-tuned":[123],"analyze":[127],"their":[128],"performances":[129],"reveal":[131],"shortcomings.":[132],"COLLIE":[133],"is":[134],"designed":[135],"extensible":[138],"lightweight,":[140],"hope":[143],"community":[145],"finds":[146],"it":[147],"useful":[148],"more":[151],"complex":[152],"evaluations":[155],"future.":[158]},"counts_by_year":[{"year":2024,"cited_by_count":2}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2023-07-19T00:00:00"}