{"id":"https://openalex.org/W4410090374","doi":"https://doi.org/10.1145/3696410.3714864","title":"Relying on the Metrics of Evaluated Agents","display_name":"Relying on the Metrics of Evaluated Agents","publication_year":2025,"publication_date":"2025-04-22","ids":{"openalex":"https://openalex.org/W4410090374","doi":"https://doi.org/10.1145/3696410.3714864"},"language":"en","primary_location":{"id":"doi:10.1145/3696410.3714864","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3696410.3714864","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3696410.3714864","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Web Conference 2025","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3696410.3714864","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101720297","display_name":"Serena Wang","orcid":"https://orcid.org/0000-0001-9664-4609"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Serena Wang","raw_affiliation_strings":["Google Research, Mountain View, CA, USA and Harvard University, Cambridge, MA, USA"],"affiliations":[{"raw_affiliation_string":"Google Research, Mountain View, CA, USA and Harvard University, Cambridge, MA, USA","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I136199984"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049812527","display_name":"Michael I. Jordan","orcid":"https://orcid.org/0000-0001-8935-817X"},"institutions":[{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["FR","US"],"is_corresponding":false,"raw_author_name":"Michael Jordan","raw_affiliation_strings":["University of California, Berkeley, Berkeley, CA, USA and Inria, Paris, France"],"affiliations":[{"raw_affiliation_string":"University of California, Berkeley, Berkeley, CA, USA and Inria, Paris, France","institution_ids":["https://openalex.org/I1326498283","https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076701535","display_name":"Katrina Ligett","orcid":"https://orcid.org/0000-0003-2780-6656"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I197251160","display_name":"Hebrew University of Jerusalem","ror":"https://ror.org/03qxff017","country_code":"IL","type":"education","lineage":["https://openalex.org/I197251160"]}],"countries":["IL","US"],"is_corresponding":false,"raw_author_name":"Katrina Ligett","raw_affiliation_strings":["Hebrew University of Jerusalem, Jerusalem, Israel and Google Research, Mountain View, CA, USA"],"affiliations":[{"raw_affiliation_string":"Hebrew University of Jerusalem, Jerusalem, Israel and Google Research, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I197251160"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5089940510","display_name":"R. Preston McAfee","orcid":"https://orcid.org/0000-0002-2587-3161"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"R. Preston McAfee","raw_affiliation_strings":["Google Research, Mountain View, CA, USA"],"affiliations":[{"raw_affiliation_string":"Google Research, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5101720297"],"corresponding_institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I136199984"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.04229792,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1468","last_page":"1487"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5967774391174316}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5967774391174316}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3696410.3714864","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3696410.3714864","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3696410.3714864","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Web Conference 2025","raw_type":"proceedings-article"},{"id":"pmh:oai:RePEc:arx:papers:2402.14005","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"preprint"}],"best_oa_location":{"id":"doi:10.1145/3696410.3714864","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3696410.3714864","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3696410.3714864","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Web Conference 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4410090374.pdf","grobid_xml":"https://content.openalex.org/works/W4410090374.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W1094775615","https://openalex.org/W1560025682","https://openalex.org/W1666623353","https://openalex.org/W1873763122","https://openalex.org/W1973465190","https://openalex.org/W2013823004","https://openalex.org/W2026559303","https://openalex.org/W2046041598","https://openalex.org/W2052312648","https://openalex.org/W2136431797","https://openalex.org/W2149902237","https://openalex.org/W2156307401","https://openalex.org/W2168432970","https://openalex.org/W2245160765","https://openalex.org/W2247195495","https://openalex.org/W2946755081","https://openalex.org/W3121860146","https://openalex.org/W3121973798","https://openalex.org/W3125449324","https://openalex.org/W3125675149","https://openalex.org/W3125682910","https://openalex.org/W3140978699","https://openalex.org/W4246986481","https://openalex.org/W4254483487","https://openalex.org/W4255261106","https://openalex.org/W4256152628","https://openalex.org/W4300764825","https://openalex.org/W4401864323"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Online":[0],"platforms":[1],"and":[2,17,120,146,164],"regulators":[3],"face":[4,49],"a":[5,96,137],"continuing":[6],"problem":[7,28],"of":[8,29,38,45,50,95],"designing":[9],"effective":[10],"evaluation":[11],"metrics.":[12],"While":[13],"tools":[14],"for":[15,160],"collecting":[16],"processing":[18],"data":[19,171],"continue":[20],"to":[21,56,68,91,98,108,135,154],"progress,":[22],"this":[23,51,75],"has":[24],"not":[25],"addressed":[26],"the":[27,39,43,48,57,93,113,118,125,131,162],"unknown":[30],"unknowns,":[31],"or":[32],"fundamental":[33],"informational":[34,52],"limitations":[35],"on":[36,170],"part":[37],"evaluator.":[40,165],"To":[41],"guide":[42],"choice":[44],"metrics":[46,110,122],"in":[47],"problem,":[53],"we":[54,82],"turn":[55],"evaluated":[58],"agents":[59],"themselves,":[60],"who":[61],"may":[62],"have":[63,88],"more":[64],"information":[65],"about":[66],"how":[67],"measure":[69],"their":[70,99],"own":[71],"outcomes.":[72],"We":[73,101,127,166],"model":[74],"interaction":[76],"as":[77],"an":[78,86,89,104,151],"agency":[79],"game,":[80],"where":[81],"ask:":[83],"When":[84],"does":[85],"agent":[87,105,132,163],"incentive":[90],"reveal":[92,109,136],"observability":[94],"metric":[97,138],"evaluator?":[100],"show":[102,129],"that":[103,111,123,130,156],"will":[106],"prefer":[107,134],"differentiate":[112,124],"most":[114],"difficult":[115],"tasks":[116],"from":[117,172],"rest,":[119],"conceal":[121],"easiest.":[126],"further":[128],"can":[133],"*garbled*":[139],"with":[140],"noise":[141],"over":[142],"both":[143,161],"fully":[144,147],"concealing":[145],"revealing.":[148],"This":[149],"indicates":[150],"economic":[152],"value":[153],"privacy":[155],"yields":[157],"Pareto":[158],"improvement":[159],"demonstrate":[167],"these":[168],"findings":[169],"online":[173],"rideshare":[174],"platforms.":[175]},"counts_by_year":[],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2025-10-10T00:00:00"}