{"id":"https://openalex.org/W3041552504","doi":"https://doi.org/10.24963/ijcai.2020/723","title":"Incentivizing Evaluation with Peer Prediction and Limited Access to Ground Truth (Extended Abstract)","display_name":"Incentivizing Evaluation with Peer Prediction and Limited Access to Ground Truth (Extended Abstract)","publication_year":2020,"publication_date":"2020-07-01","ids":{"openalex":"https://openalex.org/W3041552504","doi":"https://doi.org/10.24963/ijcai.2020/723","mag":"3041552504"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2020/723","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2020/723","pdf_url":"https://www.ijcai.org/proceedings/2020/0723.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.ijcai.org/proceedings/2020/0723.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5022280555","display_name":"Alice L. L. Gao","orcid":null},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Alice Gao","raw_affiliation_strings":["University of Waterloo, Waterloo, ON, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo, Waterloo, ON, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103266506","display_name":"James R. Wright","orcid":"https://orcid.org/0000-0001-9622-5842"},"institutions":[{"id":"https://openalex.org/I154425047","display_name":"University of Alberta","ror":"https://ror.org/0160cpw27","country_code":"CA","type":"education","lineage":["https://openalex.org/I154425047"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"James Wright","raw_affiliation_strings":["University of Alberta, Edmonton, AB, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Alberta, Edmonton, AB, Canada","institution_ids":["https://openalex.org/I154425047"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5089807699","display_name":"Kevin Leyton\u2010Brown","orcid":"https://orcid.org/0000-0002-7644-5327"},"institutions":[{"id":"https://openalex.org/I141945490","display_name":"University of British Columbia","ror":"https://ror.org/03rmrcq20","country_code":"CA","type":"education","lineage":["https://openalex.org/I141945490"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Kevin Leyton-Brown","raw_affiliation_strings":["University of British Columbia, Vancouver, BC, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of British Columbia, Vancouver, BC, Canada","institution_ids":["https://openalex.org/I141945490"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2467,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.64030771,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"5140","last_page":"5144"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9898999929428101,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/truth-telling","display_name":"Truth telling","score":0.8932949900627136},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.7828361988067627},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7551689743995667},{"id":"https://openalex.org/keywords/incentive","display_name":"Incentive","score":0.6628947257995605},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.5443375706672668},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.5248451232910156},{"id":"https://openalex.org/keywords/common-ground","display_name":"Common ground","score":0.4651196002960205},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4412001371383667},{"id":"https://openalex.org/keywords/incentive-compatibility","display_name":"Incentive compatibility","score":0.42525508999824524},{"id":"https://openalex.org/keywords/nash-equilibrium","display_name":"Nash equilibrium","score":0.4150838255882263},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.31093358993530273},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.2044934332370758},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10547512769699097},{"id":"https://openalex.org/keywords/microeconomics","display_name":"Microeconomics","score":0.08342930674552917}],"concepts":[{"id":"https://openalex.org/C2910433854","wikidata":"https://www.wikidata.org/wiki/Q1501336","display_name":"Truth telling","level":2,"score":0.8932949900627136},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.7828361988067627},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7551689743995667},{"id":"https://openalex.org/C29122968","wikidata":"https://www.wikidata.org/wiki/Q1414816","display_name":"Incentive","level":2,"score":0.6628947257995605},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.5443375706672668},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.5248451232910156},{"id":"https://openalex.org/C2777877512","wikidata":"https://www.wikidata.org/wiki/Q1116097","display_name":"Common ground","level":2,"score":0.4651196002960205},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4412001371383667},{"id":"https://openalex.org/C91810955","wikidata":"https://www.wikidata.org/wiki/Q7731670","display_name":"Incentive compatibility","level":3,"score":0.42525508999824524},{"id":"https://openalex.org/C46814582","wikidata":"https://www.wikidata.org/wiki/Q23389","display_name":"Nash equilibrium","level":2,"score":0.4150838255882263},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31093358993530273},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2044934332370758},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10547512769699097},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.08342930674552917},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C11171543","wikidata":"https://www.wikidata.org/wiki/Q41630","display_name":"Psychoanalysis","level":1,"score":0.0},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.24963/ijcai.2020/723","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2020/723","pdf_url":"https://www.ijcai.org/proceedings/2020/0723.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2020/723","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2020/723","pdf_url":"https://www.ijcai.org/proceedings/2020/0723.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.5199999809265137}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3041552504.pdf","grobid_xml":"https://content.openalex.org/works/W3041552504.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W5354901","https://openalex.org/W793737040","https://openalex.org/W1499884756","https://openalex.org/W1506509256","https://openalex.org/W1595524969","https://openalex.org/W2057283903","https://openalex.org/W2063782051","https://openalex.org/W2076435282","https://openalex.org/W2100267603","https://openalex.org/W2109304399","https://openalex.org/W2120898858","https://openalex.org/W2147498513","https://openalex.org/W2153209865","https://openalex.org/W2166164009","https://openalex.org/W2182568424","https://openalex.org/W2184706947","https://openalex.org/W2219308446","https://openalex.org/W2256714492","https://openalex.org/W2267283990","https://openalex.org/W2288655875","https://openalex.org/W2295734738","https://openalex.org/W2311017751","https://openalex.org/W2945134239","https://openalex.org/W2952721825","https://openalex.org/W2954631080","https://openalex.org/W2964177565","https://openalex.org/W3106358908","https://openalex.org/W6666620916","https://openalex.org/W6678166761","https://openalex.org/W6818947976","https://openalex.org/W6893374194"],"related_works":["https://openalex.org/W1546315632","https://openalex.org/W1901124962","https://openalex.org/W2486181377","https://openalex.org/W2127143387","https://openalex.org/W3111962505","https://openalex.org/W2607325685","https://openalex.org/W8712222","https://openalex.org/W4307535225","https://openalex.org/W2905544777","https://openalex.org/W2170515285"],"abstract_inverted_index":{"In":[0],"many":[1],"settings,":[2],"an":[3],"effective":[4],"way":[5],"of":[6,9,131,165,223],"evaluating":[7,31],"objects":[8],"interest":[10],"is":[11,44,62,106,143,176,181],"to":[12,19,45,48,54,80,91,97,112,139,216],"collect":[13],"evaluations":[14,22,51],"from":[15],"dispersed":[16],"individuals":[17],"and":[18,30,53,94,196],"aggregate":[20],"these":[21],"together.":[23],"Some":[24],"examples":[25],"are":[26,110],"categorizing":[27],"online":[28],"content":[29],"student":[32],"assignments":[33],"via":[34],"peer":[35,157],"grading.":[36],"For":[37],"this":[38,104,133],"data":[39],"science":[40],"problem,":[41],"one":[42],"challenge":[43],"motivate":[46],"participants":[47],"conduct":[49],"such":[50,173],"carefully":[52],"report":[55],"them":[56],"honestly,":[57],"particularly":[58],"when":[59,147,172],"doing":[60],"so":[61],"costly.":[63],"Existing":[64],"approaches,":[65],"notably":[66],"peer-prediction":[67,224],"mechanisms,":[68],"can":[69,160],"incentivize":[70],"truth":[71,175,192,218],"telling":[72],"in":[73,82,145,155],"equilibrium.":[74],"However,":[75],"they":[76],"also":[77,182],"give":[78],"rise":[79],"equilibria":[81],"which":[83,142],"agents":[84,109],"do":[85],"not":[86],"pay":[87],"the":[88,118,156,199,206],"costs":[89],"required":[90],"evaluate":[92],"accurately,":[93],"hence":[95],"fail":[96],"elicit":[98],"useful":[99],"information.":[100],"We":[101,127],"show":[102,204],"that":[103,205],"problem":[105,134],"unavoidable":[107],"whenever":[108],"able":[111],"coordinate":[113],"using":[114],"low-cost":[115],"signals":[116],"about":[117],"items":[119],"being":[120],"evaluated":[121],"(e.g.,":[122],"text":[123],"labels":[124],"or":[125],"pictures).":[126],"then":[128],"consider":[129],"ways":[130],"circumventing":[132],"by":[135],"comparing":[136],"agents'":[137],"reports":[138],"ground":[140,174,191,217],"truth,":[141],"available":[144],"practice":[146],"there":[148],"exist":[149],"trusted":[150],"evaluators---such":[151],"as":[152],"teaching":[153],"assistants":[154],"grading":[158],"scenario---who":[159],"perform":[161],"a":[162,178,220],"limited":[163],"number":[164],"unbiased":[166],"(but":[167],"noisy)":[168],"evaluations.":[169],"Of":[170],"course,":[171],"available,":[177],"simpler":[179,207],"approach":[180],"possible:":[183],"rewarding":[184,198],"each":[185],"agent":[186,200],"based":[187],"on":[188],"agreement":[189],"with":[190,193],"some":[194],"probability,":[195],"unconditionally":[197],"otherwise.":[201],"Surprisingly,":[202],"we":[203],"mechanism":[208],"achieves":[209],"stronger":[210],"incentive":[211],"guarantees":[212],"given":[213],"less":[214],"access":[215],"than":[219],"large":[221],"set":[222],"mechanisms.":[225]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}