{"id":"https://openalex.org/W2275133185","doi":null,"title":"Quality Control in Crowdsourcing: An Objective Measurement Approach to Identifying and Correcting Rater Effects in the Social Evaluation of Products and Services.","display_name":"Quality Control in Crowdsourcing: An Objective Measurement Approach to Identifying and Correcting Rater Effects in the Social Evaluation of Products and Services.","publication_year":2012,"publication_date":"2012-03-23","ids":{"openalex":"https://openalex.org/W2275133185","mag":"2275133185"},"language":"en","primary_location":{"id":"mag:2275133185","doi":null,"is_oa":false,"landing_page_url":"http://renselange.com/pdf/1-quality-control-in-crowdsourcing.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306420577","display_name":"National Conference on Artificial Intelligence","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"National Conference on Artificial Intelligence","raw_type":null},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102084027","display_name":"Rense Lange","orcid":"https://orcid.org/0000-0002-6798-3772","observed_orcids":["https://orcid.org/0000-0002-6798-3772"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rense Lange","raw_affiliation_strings":["Integrated Knowledge Systems"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Integrated Knowledge Systems","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077795894","display_name":"Xavier Lange","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xavier Lange","raw_affiliation_strings":["Tureus Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tureus Inc","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.1909,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.40227137,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":96},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9413999915122986,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9413999915122986,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.03220000118017197,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.0024999999441206455,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"crowdsourcing","score":0.871999979019165},{"id":"https://openalex.org/keywords/rater-effects","display_name":"rater effects","score":0.8069999814033508},{"id":"https://openalex.org/keywords/rasch-scaling","display_name":"Rasch scaling","score":0.8230000138282776},{"id":"https://openalex.org/keywords/product-ratings","display_name":"product ratings","score":0.6650000214576721},{"id":"https://openalex.org/keywords/customer-feedback","display_name":"customer feedback","score":0.7940000295639038},{"id":"https://openalex.org/keywords/rating-reliability","display_name":"rating reliability","score":0.3569999933242798}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9629075527191162},{"id":"https://openalex.org/C101266164","wikidata":"https://www.wikidata.org/wiki/Q2131821","display_name":"Rasch model","level":2,"score":0.8738088607788086},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.6525751352310181},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.64693284034729},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6180861592292786},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.5416711568832397},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5358193516731262},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.43441981077194214},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.35314735770225525},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2166326940059662},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.1785869300365448},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.16395044326782227},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.09772193431854248},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:2275133185","doi":null,"is_oa":false,"landing_page_url":"http://renselange.com/pdf/1-quality-control-in-crowdsourcing.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306420577","display_name":"National Conference on Artificial Intelligence","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"National Conference on Artificial Intelligence","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W51759634","https://openalex.org/W134167318","https://openalex.org/W265329213","https://openalex.org/W618991135","https://openalex.org/W657332045","https://openalex.org/W1991623719","https://openalex.org/W1995617856","https://openalex.org/W2109463481","https://openalex.org/W2127374659","https://openalex.org/W2130713932","https://openalex.org/W2149489787","https://openalex.org/W2162729082"],"related_works":["https://openalex.org/W1985063974","https://openalex.org/W3112760125","https://openalex.org/W2614452370","https://openalex.org/W2180780611","https://openalex.org/W2069396035","https://openalex.org/W2337667705","https://openalex.org/W2036431079","https://openalex.org/W3177059608","https://openalex.org/W3088965898","https://openalex.org/W6903790299"],"abstract_inverted_index":{"Crowdsourcing":[0],"requires":[1],"new":[2],"strategies":[3],"to":[4,30],"evaluate":[5],"the":[6,12,32,77,88],"workers":[7,35,107],"involved":[8],"as":[9,11,26,75,83,101],"well":[10,102],"quality":[13,98],"of":[14,34,51,63,79,87],"workers\u2019":[15],"output.":[16],"Using":[17],"customer":[18],"feedback":[19,116],"data,":[20],"we":[21,68,103],"introduce":[22],"multi-facetted":[23],"Rasch":[24,94],"scaling":[25,95],"an":[27],"evaluation":[28],"technique":[29],"assess":[31],"contributions":[33],"and":[36,108],"products":[37,109],"simultaneously":[38],"within":[39],"a":[40,48,61],"single":[41],"coherent":[42],"measurement":[43],"framework.":[44],"Based":[46],"on":[47],"data":[49],"set":[50],"about":[52],"250,000":[53],"customers":[54],"who":[55],"rated":[56],"nearly":[57,64],"115,000":[58],"products,":[59],"for":[60],"total":[62],"3":[65],"million":[66],"cases,":[67],"found":[69],"that":[70,114],"product":[71],"ratings":[72],"reflect":[73],"almost":[74],"much":[76],"existence":[78],"stable":[80],"rater":[81],"differences":[82],"they":[84],"are":[85],"indicative":[86],"products\u2019":[89],"properties.":[90],"We":[91],"illustrate":[92],"how":[93,105],"provides":[96],"extensive":[97],"control":[99],"mechanisms;":[100],"show":[104],"aberrant":[106],"can":[110,120],"be":[111,121],"identified":[112],"so":[113],"appropriate":[115],"and/or":[117],"corrective":[118],"actions":[119],"initiated.":[122]},"counts_by_year":[{"year":2019,"cited_by_count":2},{"year":2017,"cited_by_count":2},{"year":2015,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}