{"id":"https://openalex.org/W3041687329","doi":"https://doi.org/10.1145/3391403.3399488","title":"Surrogate Scoring Rules","display_name":"Surrogate Scoring Rules","publication_year":2020,"publication_date":"2020-07-09","ids":{"openalex":"https://openalex.org/W3041687329","doi":"https://doi.org/10.1145/3391403.3399488","mag":"3041687329"},"language":"en","primary_location":{"id":"doi:10.1145/3391403.3399488","doi":"https://doi.org/10.1145/3391403.3399488","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3391403.3399488","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3391403.3399488","source":{"id":"https://openalex.org/S4363608973","display_name":"ACM Conference on Economics and Computation (EC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st ACM Conference on Economics and Computation","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3391403.3399488","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030965346","display_name":"Liu Yang","orcid":"https://orcid.org/0000-0002-6015-6141","observed_orcids":["https://orcid.org/0000-0002-6015-6141"]},"institutions":[{"id":"https://openalex.org/I185103710","display_name":"University of California, Santa Cruz","ror":"https://ror.org/03s65by71","country_code":"US","type":"education","lineage":["https://openalex.org/I185103710"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang Liu","raw_affiliation_strings":["University of California, Santa Cruz, Santa Cruz, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California, Santa Cruz, Santa Cruz, CA, USA","institution_ids":["https://openalex.org/I185103710"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100612392","display_name":"Juntao Wang","orcid":"https://orcid.org/0000-0002-4072-9870","observed_orcids":["https://orcid.org/0000-0002-4072-9870"]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Juntao Wang","raw_affiliation_strings":["Harvard University, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University, Cambridge, MA, USA","institution_ids":["https://openalex.org/I136199984"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100738419","display_name":"Yiling Chen","orcid":"https://orcid.org/0000-0002-0166-594X","observed_orcids":["https://orcid.org/0000-0002-0166-594X"]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yiling Chen","raw_affiliation_strings":["Harvard University, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University, Cambridge, MA, USA","institution_ids":["https://openalex.org/I136199984"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.1672,"has_fulltext":true,"cited_by_count":29,"citation_normalized_percentile":{"value":0.98729991,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"853","last_page":"871"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11031","display_name":"Game Theory and Applications","score":0.5687999725341797,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11031","display_name":"Game Theory and Applications","score":0.5687999725341797,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.2101999968290329,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10991","display_name":"Game Theory and Voting Systems","score":0.08389999717473984,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/strictly-proper-scoring-rules","display_name":"strictly proper scoring rules","score":0.9959999918937683},{"id":"https://openalex.org/keywords/incentive-compatibility","display_name":"incentive compatibility","score":0.6949999928474426},{"id":"https://openalex.org/keywords/belief-elicitation","display_name":"belief elicitation","score":0.6230000257492065},{"id":"https://openalex.org/keywords/probabilistic-beliefs","display_name":"probabilistic beliefs","score":0.6029999852180481},{"id":"https://openalex.org/keywords/bias-correction","display_name":"bias correction","score":0.746999979019165},{"id":"https://openalex.org/keywords/error-rate-estimation","display_name":"error rate estimation","score":0.9430000185966492},{"id":"https://openalex.org/keywords/multi-task-prediction","display_name":"multi-task prediction","score":0.6050000190734863}],"concepts":[{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6890193819999695},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6887749433517456},{"id":"https://openalex.org/C63002673","wikidata":"https://www.wikidata.org/wiki/Q2260590","display_name":"Scoring rule","level":2,"score":0.6657487154006958},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6293566823005676},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.6171882152557373},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5983284711837769},{"id":"https://openalex.org/C2781089630","wikidata":"https://www.wikidata.org/wiki/Q21856745","display_name":"Realization (probability)","level":2,"score":0.5662108659744263},{"id":"https://openalex.org/C122123141","wikidata":"https://www.wikidata.org/wiki/Q176623","display_name":"Random variable","level":2,"score":0.5582627654075623},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49056190252304077},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.4881508946418762},{"id":"https://openalex.org/C144559511","wikidata":"https://www.wikidata.org/wiki/Q2986279","display_name":"Principal (computer security)","level":2,"score":0.47233787178993225},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.46497273445129395},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.46347957849502563},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4217678904533386},{"id":"https://openalex.org/C152139883","wikidata":"https://www.wikidata.org/wiki/Q252973","display_name":"Mutual information","level":2,"score":0.4118140935897827},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.21142986416816711},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.18735343217849731},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3391403.3399488","doi":"https://doi.org/10.1145/3391403.3399488","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3391403.3399488","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3391403.3399488","source":{"id":"https://openalex.org/S4363608973","display_name":"ACM Conference on Economics and Computation (EC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st ACM Conference on Economics and Computation","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3391403.3399488","doi":"https://doi.org/10.1145/3391403.3399488","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3391403.3399488","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3391403.3399488","source":{"id":"https://openalex.org/S4363608973","display_name":"ACM Conference on Economics and Computation (EC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 21st ACM Conference on Economics and Computation","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2448990042","display_name":null,"funder_award_id":"Contract No. N66001-19-C-4014","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G4298657579","display_name":null,"funder_award_id":"2017-17061500006","funder_id":"https://openalex.org/F4320333051","funder_display_name":"Intelligence Advanced Research Projects Activity"},{"id":"https://openalex.org/G6708036262","display_name":null,"funder_award_id":"N66001-19-C-4014","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G762897638","display_name":"AF: Small: Learning and Optimization with Strategic Data Sources","funder_award_id":"1718549","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8215844777","display_name":null,"funder_award_id":"N66001-19","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320312530","display_name":"Office of the Director of National Intelligence","ror":"https://ror.org/01v3fsc55"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320332815","display_name":"Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320333051","display_name":"Intelligence Advanced Research Projects Activity","ror":"https://ror.org/01v3fsc55"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3041687329.pdf","grobid_xml":"https://content.openalex.org/works/W3041687329.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W793737040","https://openalex.org/W1489441240","https://openalex.org/W1998274472","https://openalex.org/W2002471330","https://openalex.org/W2025720061","https://openalex.org/W2029552385","https://openalex.org/W2041268952","https://openalex.org/W2073241381","https://openalex.org/W2107189314","https://openalex.org/W2113290770","https://openalex.org/W2120898858","https://openalex.org/W2167460663","https://openalex.org/W2202525882","https://openalex.org/W2241862190","https://openalex.org/W2267283990","https://openalex.org/W2295734738","https://openalex.org/W2309724559","https://openalex.org/W2418640254","https://openalex.org/W2610265803","https://openalex.org/W2901118154","https://openalex.org/W2954631080","https://openalex.org/W2963319311","https://openalex.org/W2963877745","https://openalex.org/W2978435634","https://openalex.org/W3001359517","https://openalex.org/W3006251828"],"related_works":["https://openalex.org/W2984090571","https://openalex.org/W2900320994","https://openalex.org/W2788948583","https://openalex.org/W3126015563","https://openalex.org/W4383557931","https://openalex.org/W2902462695","https://openalex.org/W2135503447","https://openalex.org/W7095982006","https://openalex.org/W2611075842","https://openalex.org/W4299804667"],"abstract_inverted_index":{"Strictly":[0],"proper":[1],"scoring":[2,53],"rules":[3,54],"(SPSR)":[4],"are":[5],"incentive":[6],"compatible":[7],"for":[8,92,158],"eliciting":[9],"information":[10,108,147],"about":[11,109],"random":[12,28,115],"variables":[13],"from":[14,184],"strategic":[15],"agents":[16,22],"when":[17],"the":[18,24,27,33,110,114,131,144,149,159],"principal":[19,59],"can":[20],"reward":[21],"after":[23],"realization":[25],"of":[26,35,107,113,138,146,151],"variables.":[29],"They":[30],"also":[31],"quantify":[32,143],"quality":[34,145],"elicited":[36],"information,":[37],"with":[38,103,161],"more":[39],"accurate":[40],"predictions":[41],"receiving":[42],"higher":[43],"scores":[44],"in":[45,118,124,171],"expectation.":[46],"In":[47],"this":[48],"paper,":[49],"we":[50],"extend":[51],"such":[52],"to":[55,68,130],"settings":[56],"where":[57],"a":[58,82,93,104,119,135,165],"elicits":[60],"private":[61],"probabilistic":[62],"beliefs":[63],"but":[64],"only":[65],"has":[66],"access":[67,129],"agents'":[69,98],"reports.":[70,99],"We":[71,100],"name":[72],"our":[73],"solution":[74],"Surrogate":[75],"Scoring":[76],"Rules":[77],"(SSR).":[78],"SSR":[79,117,139,167],"build":[80],"on":[81],"bias":[83],"correction":[84],"step":[85],"and":[86,179],"an":[87],"error":[88],"rate":[89],"estimation":[90],"procedure":[91],"reference":[94],"answer":[95],"defined":[96],"using":[97,181],"show":[101],"that,":[102],"single":[105],"bit":[106],"prior":[111],"distribution":[112],"variables,":[116],"multi-task":[120],"setting":[121,160],"recover":[122],"SPSR":[123,156],"expectation,":[125],"as":[126,155],"if":[127],"having":[128],"ground":[132,152,162],"truth.":[133,163],"Therefore,":[134],"salient":[136],"feature":[137],"is":[140,175],"that":[141],"they":[142],"despite":[148],"lack":[150],"truth,":[153],"just":[154],"do":[157],"As":[164],"by-product,":[166],"induce":[168],"dominant":[169],"truthfulness":[170],"reporting.":[172],"Our":[173],"method":[174],"verified":[176],"both":[177],"theoretically":[178],"empirically":[180],"data":[182],"collected":[183],"real":[185],"human":[186],"forecasters.":[187]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":11},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":4}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2020-07-16T00:00:00"}