{"id":"https://openalex.org/W4360891268","doi":"https://doi.org/10.48550/arxiv.2303.13360","title":"Towards the Scalable Evaluation of Cooperativeness in Language Models","display_name":"Towards the Scalable Evaluation of Cooperativeness in Language Models","publication_year":2023,"publication_date":"2023-03-16","ids":{"openalex":"https://openalex.org/W4360891268","doi":"https://doi.org/10.48550/arxiv.2303.13360"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2303.13360","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2303.13360","pdf_url":"https://arxiv.org/pdf/2303.13360","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2303.13360","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058159907","display_name":"Alan H. S. Chan","orcid":"https://orcid.org/0000-0002-7541-3163"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Chan, Alan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085895682","display_name":"Maxime Rich\u00e9","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rich\u00e9, Maxime","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5010989287","display_name":"Jesse Clifton","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Clifton, Jesse","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5058159907"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9399999976158142,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9366000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cooperativeness","display_name":"Cooperativeness","score":0.9526654481887817},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.753838300704956},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.5845203995704651},{"id":"https://openalex.org/keywords/negotiation","display_name":"Negotiation","score":0.5836949348449707},{"id":"https://openalex.org/keywords/resolution","display_name":"Resolution (logic)","score":0.5337874293327332},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5151106715202332},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.49752119183540344},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4335818588733673},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4201887547969818},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.14705300331115723},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.11880317330360413},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.09447720646858215},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.0909634530544281}],"concepts":[{"id":"https://openalex.org/C2781293773","wikidata":"https://www.wikidata.org/wiki/Q5167926","display_name":"Cooperativeness","level":4,"score":0.9526654481887817},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.753838300704956},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.5845203995704651},{"id":"https://openalex.org/C199776023","wikidata":"https://www.wikidata.org/wiki/Q202875","display_name":"Negotiation","level":2,"score":0.5836949348449707},{"id":"https://openalex.org/C138268822","wikidata":"https://www.wikidata.org/wiki/Q1051925","display_name":"Resolution (logic)","level":2,"score":0.5337874293327332},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5151106715202332},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.49752119183540344},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4335818588733673},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4201887547969818},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.14705300331115723},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.11880317330360413},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.09447720646858215},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0909634530544281},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C187288502","wikidata":"https://www.wikidata.org/wiki/Q641118","display_name":"Personality","level":2,"score":0.0},{"id":"https://openalex.org/C61644593","wikidata":"https://www.wikidata.org/wiki/Q80157","display_name":"Temperament","level":3,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2303.13360","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2303.13360","pdf_url":"https://arxiv.org/pdf/2303.13360","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2303.13360","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2303.13360","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2303.13360","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2303.13360","pdf_url":"https://arxiv.org/pdf/2303.13360","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"score":0.49000000953674316,"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4360891268.pdf","grobid_xml":"https://content.openalex.org/works/W4360891268.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2017209806","https://openalex.org/W3124772035","https://openalex.org/W2288879570","https://openalex.org/W3122259054","https://openalex.org/W2762049957","https://openalex.org/W1516665276","https://openalex.org/W2258823252","https://openalex.org/W4200155547","https://openalex.org/W2099492212","https://openalex.org/W2364524776"],"abstract_inverted_index":{"It":[0],"is":[1,58],"likely":[2],"that":[3,130,197,206],"AI":[4,37],"systems":[5],"driven":[6],"by":[7],"pre-trained":[8],"language":[9,92,122,149],"models":[10,199,217],"(PLMs)":[11],"will":[12],"increasingly":[13],"be":[14,137,208],"used":[15],"to":[16,41,111,124,136,151,201,219],"assist":[17],"humans":[18],"in":[19,50,71,105,139,203],"high-stakes":[20],"interactions":[21],"with":[22,32,84,87,157],"other":[23,216],"agents,":[24],"such":[25,126],"as":[26,97,210],"negotiation":[27],"or":[28],"conflict":[29],"resolution.":[30],"Consistent":[31],"the":[33,45,59,106,131,167],"goals":[34],"of":[35,48,61,108,133,174,188],"Cooperative":[36],"\\citep{dafoe_open_2020},":[38],"we":[39,79,100,116,170],"wish":[40],"understand":[42],"and":[43,90,120,147,185,190],"shape":[44],"multi-agent":[46],"behaviors":[47],"PLMs":[49],"a":[51,91,121,148,172,204],"pro-social":[52],"manner.":[53],"An":[54],"important":[55],"first":[56],"step":[57],"evaluation":[60],"model":[62,123,150],"behaviour":[63,70],"across":[64],"diverse":[65],"cooperation":[66],"problems.":[67],"Since":[68],"desired":[69],"an":[72],"interaction":[73],"depends":[74],"upon":[75],"precise":[76],"game-theoretic":[77,113,160],"structure,":[78,161],"focus":[80],"on":[81,166,177,192],"generating":[82],"scenarios":[83,109,155],"particular":[85,112],"structures":[86],"both":[88,118,140,145,183],"crowdworkers":[89,119,146],"model.":[93],"Our":[94],"work":[95],"proceeds":[96],"follows.":[98],"First,":[99],"discuss":[101],"key":[102],"methodological":[103],"issues":[104],"generation":[107],"corresponding":[110],"structures.":[114],"Second,":[115],"employ":[117],"generate":[125],"scenarios.":[127],"We":[128,142,181,195],"find":[129,196],"quality":[132],"generations":[134],"tends":[135],"mediocre":[138],"cases.":[141],"additionally":[143],"get":[144],"judge":[152],"whether":[153],"given":[154],"align":[156],"their":[158],"intended":[159],"finding":[162],"mixed":[163],"results":[164],"depending":[165],"game.":[168],"Third,":[169],"provide":[171,182],"dataset":[173],"scenario":[175],"based":[176],"our":[178],"data":[179],"generated.":[180],"quantitative":[184],"qualitative":[186],"evaluations":[187],"UnifiedQA":[189],"GPT-3":[191],"this":[193],"dataset.":[194],"instruct-tuned":[198],"tend":[200],"act":[202],"way":[205],"could":[207],"perceived":[209],"cooperative":[211],"when":[212],"scaled":[213],"up,":[214],"while":[215],"seemed":[218],"have":[220],"flat":[221],"scaling":[222],"trends.":[223]},"counts_by_year":[],"updated_date":"2026-03-08T08:50:53.379069","created_date":"2025-10-10T00:00:00"}