{"id":"https://openalex.org/W2147498513","doi":"https://doi.org/10.1609/hcomp.v1i1.13089","title":"Dwelling on the Negative: Incentivizing Effort in Peer Prediction","display_name":"Dwelling on the Negative: Incentivizing Effort in Peer Prediction","publication_year":2013,"publication_date":"2013-11-03","ids":{"openalex":"https://openalex.org/W2147498513","doi":"https://doi.org/10.1609/hcomp.v1i1.13089","mag":"2147498513"},"language":"en","primary_location":{"id":"doi:10.1609/hcomp.v1i1.13089","is_oa":true,"landing_page_url":"https://doi.org/10.1609/hcomp.v1i1.13089","pdf_url":"https://ojs.aaai.org/index.php/HCOMP/article/download/13089/12937","source":{"id":"https://openalex.org/S4387284122","display_name":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","issn_l":"2769-1330","issn":["2769-1330","2769-1349"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/HCOMP/article/download/13089/12937","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086535574","display_name":"Jens Witkowski","orcid":"https://orcid.org/0000-0002-4748-9099"},"institutions":[{"id":"https://openalex.org/I161046081","display_name":"University of Freiburg","ror":"https://ror.org/0245cg223","country_code":"DE","type":"education","lineage":["https://openalex.org/I161046081"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Jens Witkowski","raw_affiliation_strings":["Albert-Ludwigs-Universit\u00e4t Freiburg"],"affiliations":[{"raw_affiliation_string":"Albert-Ludwigs-Universit\u00e4t Freiburg","institution_ids":["https://openalex.org/I161046081"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062949033","display_name":"Yoram Bachrach","orcid":"https://orcid.org/0000-0002-4382-7636"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Yoram Bachrach","raw_affiliation_strings":["Microsoft Research Cambridge"],"affiliations":[{"raw_affiliation_string":"Microsoft Research Cambridge","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053550912","display_name":"Peter Key","orcid":"https://orcid.org/0000-0003-4799-368X"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Peter Key","raw_affiliation_strings":["Microsoft Research Cambridge"],"affiliations":[{"raw_affiliation_string":"Microsoft Research Cambridge","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5086173064","display_name":"David C. Parkes","orcid":"https://orcid.org/0000-0002-2701-3464"},"institutions":[{"id":"https://openalex.org/I2801851002","display_name":"Harvard University Press","ror":"https://ror.org/006v7bf86","country_code":"US","type":"other","lineage":["https://openalex.org/I136199984","https://openalex.org/I2801851002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Parkes","raw_affiliation_strings":["Harvard University"],"affiliations":[{"raw_affiliation_string":"Harvard University","institution_ids":["https://openalex.org/I2801851002"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5086535574"],"corresponding_institution_ids":["https://openalex.org/I161046081"],"apc_list":null,"apc_paid":null,"fwci":11.1262,"has_fulltext":true,"cited_by_count":64,"citation_normalized_percentile":{"value":0.98512234,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"1","issue":null,"first_page":"190","last_page":"197"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10646","display_name":"Experimental Behavioral Economics Studies","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11674","display_name":"Sports Analytics and Performance","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.820682168006897},{"id":"https://openalex.org/keywords/payment","display_name":"Payment","score":0.7987663745880127},{"id":"https://openalex.org/keywords/incentive","display_name":"Incentive","score":0.7257245779037476},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.7223673462867737},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.699675440788269},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5423027873039246},{"id":"https://openalex.org/keywords/microeconomics","display_name":"Microeconomics","score":0.5383859872817993},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.4576171934604645},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.4532249867916107},{"id":"https://openalex.org/keywords/mechanism-design","display_name":"Mechanism design","score":0.4521109163761139},{"id":"https://openalex.org/keywords/actuarial-science","display_name":"Actuarial science","score":0.3682650327682495},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.3266404867172241},{"id":"https://openalex.org/keywords/risk-analysis","display_name":"Risk analysis (engineering)","score":0.32557186484336853},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.2977917194366455},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.16685721278190613},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14545735716819763}],"concepts":[{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.820682168006897},{"id":"https://openalex.org/C145097563","wikidata":"https://www.wikidata.org/wiki/Q1148747","display_name":"Payment","level":2,"score":0.7987663745880127},{"id":"https://openalex.org/C29122968","wikidata":"https://www.wikidata.org/wiki/Q1414816","display_name":"Incentive","level":2,"score":0.7257245779037476},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.7223673462867737},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.699675440788269},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5423027873039246},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.5383859872817993},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.4576171934604645},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.4532249867916107},{"id":"https://openalex.org/C153517567","wikidata":"https://www.wikidata.org/wiki/Q26090","display_name":"Mechanism design","level":2,"score":0.4521109163761139},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.3682650327682495},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.3266404867172241},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.32557186484336853},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.2977917194366455},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.16685721278190613},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14545735716819763},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1609/hcomp.v1i1.13089","is_oa":true,"landing_page_url":"https://doi.org/10.1609/hcomp.v1i1.13089","pdf_url":"https://ojs.aaai.org/index.php/HCOMP/article/download/13089/12937","source":{"id":"https://openalex.org/S4387284122","display_name":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","issn_l":"2769-1330","issn":["2769-1330","2769-1349"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1609/hcomp.v1i1.13089","is_oa":true,"landing_page_url":"https://doi.org/10.1609/hcomp.v1i1.13089","pdf_url":"https://ojs.aaai.org/index.php/HCOMP/article/download/13089/12937","source":{"id":"https://openalex.org/S4387284122","display_name":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","issn_l":"2769-1330","issn":["2769-1330","2769-1349"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Human Computation and Crowdsourcing","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7933652722","display_name":"HCC: Small: Incentive-Compatible Machine Learning","funder_award_id":"0915016","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320308943","display_name":"Microsoft Research","ror":"https://ror.org/00d0nc645"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W2147498513.pdf"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W793737040","https://openalex.org/W1496221018","https://openalex.org/W2042863768","https://openalex.org/W2063782051","https://openalex.org/W2109304399","https://openalex.org/W2120898858","https://openalex.org/W2122588206","https://openalex.org/W2138524027","https://openalex.org/W2139252194","https://openalex.org/W2153209865","https://openalex.org/W2160367301","https://openalex.org/W2187232655","https://openalex.org/W2963112694","https://openalex.org/W3106358908","https://openalex.org/W4301174940","https://openalex.org/W6629605197","https://openalex.org/W6666620916","https://openalex.org/W6677855075","https://openalex.org/W6680629092","https://openalex.org/W6687115968","https://openalex.org/W6786058386"],"related_works":["https://openalex.org/W2073713056","https://openalex.org/W3110702597","https://openalex.org/W2125620709","https://openalex.org/W2110441383","https://openalex.org/W2045932760","https://openalex.org/W4233149903","https://openalex.org/W2326878701","https://openalex.org/W4293864700","https://openalex.org/W3160516639","https://openalex.org/W2491349548"],"abstract_inverted_index":{"Agents":[0],"are":[1],"asked":[2],"to":[3,62,67,110,117,120,132,145,156,158,164],"rank":[4],"two":[5],"objects":[6],"in":[7,17,41],"a":[8,57,63,75,80,113,133],"setting":[9],"where":[10],"effort":[11],"is":[12,20,73,91],"costly":[13],"and":[14,51,96,124,166,194],"agents":[15,88,147],"differ":[16],"quality":[18,64,72,90,134,149,153],"(which":[19],"the":[21,27,42,47,103,128,152,170,185,190,198],"probability":[22],"that":[23,37,83,87,139],"they":[24],"can":[25,142,178],"identify":[26],"correct,":[28],"ground":[29],"truth,":[30],"ranking).":[31],"We":[32],"study":[33],"simple":[34],"output-agreement":[35],"mechanisms":[36],"pay":[38],"an":[39,70],"agent":[40],"case":[43],"she":[44],"agrees":[45],"with":[46,148],"report":[48],"of":[49,106,115,173],"another,":[50],"potentially":[52],"penalizes":[53],"for":[54,204],"disagreement":[55],"through":[56],"negative":[58,100,125,140],"payment.":[59],"Assuming":[60],"access":[61,131],"oracle,":[65,135],"able":[66],"determine":[68],"whether":[69],"agent's":[71],"above":[74,92,162],"given":[76],"threshold,":[77],"we":[78,136],"design":[79,176],"payment":[81],"scheme":[82],"aligns":[84],"incentives":[85],"so":[86],"whose":[89],"this":[93,107],"threshold":[94,154,177],"participate":[95,165],"invest":[97,167],"effort.":[98,168],"Precluding":[99],"payments":[101,141],"leads":[102],"expected":[104,187],"cost":[105,188],"quality-oracle":[108,192],"mechanism":[109,183],"increase":[111],"by":[112],"factor":[114],"2":[116],"5":[118],"relative":[119],"allowing":[121],"both":[122],"positive":[123],"payments.":[126],"Dropping":[127],"assumption":[129],"about":[130],"further":[137],"show":[138],"be":[143,179],"used":[144],"make":[146],"lower":[150],"than":[151],"choose":[155],"not":[157],"participate,":[159],"while":[160],"those":[161],"continue":[163],"Through":[169],"appropriate":[171],"choice":[172],"payments,":[174],"any":[175],"achieved.":[180],"This":[181],"self-selection":[182,199],"has":[184],"same":[186],"as":[189],"cost-minimal":[191],"mechanism,":[193,200],"thus":[195],"when":[196],"using":[197],"perfect":[201],"screening":[202],"comes":[203],"free.":[205]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":6},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":7},{"year":2017,"cited_by_count":8},{"year":2016,"cited_by_count":8},{"year":2015,"cited_by_count":7},{"year":2014,"cited_by_count":11}],"updated_date":"2026-03-15T09:29:46.208133","created_date":"2025-10-10T00:00:00"}