{"id":"https://openalex.org/W2899357352","doi":null,"title":"Domain-Independent Quality Measures for Crowd Truth Disagreement.","display_name":"Domain-Independent Quality Measures for Crowd Truth Disagreement.","alternate_titles":[],"publication_year":2013,"publication_date":"2013-01-01","ids":{"openalex":"https://openalex.org/W2899357352","mag":"2899357352"},"language":"en","primary_location":{"id":"mag:2899357352","doi":null,"is_oa":false,"landing_page_url":"http://ceur-ws.org/Vol-1123/paper2.pdf","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"parent_works":[],"lineage":["https://openalex.org/W2899357352"],"child_works_count":0,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5056796427","display_name":"Oana Inel","orcid":"https://orcid.org/0000-0003-4691-6586","observed_orcids":["https://orcid.org/0000-0003-4691-6586"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Oana Inel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016381010","display_name":"Lora M. Aroyo","orcid":"https://orcid.org/0000-0001-9402-1133","observed_orcids":["https://orcid.org/0000-0001-9402-1133"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Lora Aroyo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106085231","display_name":"Chris Welty","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Chris Welty","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5040841362","display_name":"Robert-Jan Sips","orcid":"https://orcid.org/0000-0002-2316-7183","observed_orcids":["https://orcid.org/0000-0002-2316-7183"]},"institutions":[],"countries":[],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Robert-Jan Sips","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.7624,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.89524182,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":86,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"2","last_page":"13"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.0006000000284984708,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"crowdsourcing","score":0.8349999785423279},{"id":"https://openalex.org/keywords/annotation-quality-assessment","display_name":"annotation quality assessment","score":0.5910000205039978},{"id":"https://openalex.org/keywords/event-extraction","display_name":"event extraction","score":0.8529999852180481},{"id":"https://openalex.org/keywords/crowdflower","display_name":"CrowdFlower","score":0.7440000176429749},{"id":"https://openalex.org/keywords/amazon-mechanical-turk","display_name":"Amazon Mechanical Turk","score":0.8679999709129333},{"id":"https://openalex.org/keywords/ground-truth-data-collection","display_name":"ground truth data collection","score":0.5799999833106995}],"concepts":[{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5219956636428833},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4679830074310303},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.42089152336120605},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36839327216148376},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.20176681876182556},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.1746285855770111},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.11880213022232056},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"mag:2899357352","doi":null,"is_oa":false,"landing_page_url":"http://ceur-ws.org/Vol-1123/paper2.pdf","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.428.8468","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.428.8468","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://ceur-ws.org/Vol-1123/paper2.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":7,"referenced_works":["https://openalex.org/W1514944708","https://openalex.org/W2003185534","https://openalex.org/W2003497265","https://openalex.org/W2056478778","https://openalex.org/W2098865355","https://openalex.org/W2146286563","https://openalex.org/W2149803936"],"related_works":["https://openalex.org/W2891077499","https://openalex.org/W2886159104","https://openalex.org/W2468363730","https://openalex.org/W1494186835","https://openalex.org/W6902228961","https://openalex.org/W2729498950","https://openalex.org/W2170538746","https://openalex.org/W2765320432","https://openalex.org/W2409069296","https://openalex.org/W2487227665"],"abstract_inverted_index":{"Abstract.":[0],"Using":[1],"crowdsourcing":[2,83],"platforms":[3],"such":[4],"as":[5],"CrowdFlower":[6],"and":[7,35,81,102,120,160,183],"Amazon":[8],"Mechanical":[9],"Turk":[10],"for":[11,30,66,79,98,148,175],"gathering":[12],"human":[13],"annotation":[14,33,48,106],"data":[15,181],"has":[16],"become":[17],"now":[18],"a":[19,44,63,145],"mainstream":[20],"process.":[21],"Such":[22],"crowd":[23],"involvement":[24],"can":[25,42,140],"reduce":[26],"the":[27,37,70,91,95,99,104,137,152,156,161,172,176,188],"time":[28],"needed":[29],"solving":[31],"an":[32],"task":[34],"with":[36],"large":[38],"number":[39],"of":[40,47,69,136,158],"annotators":[41],"be":[43,141],"valuable":[45],"source":[46],"diversity.":[49],"In":[50,72],"order":[51],"to":[52,61,154],"harness":[53,155],"this":[54,73],"diversity":[55,157],"across":[56,85],"domains":[57,86],"it":[58],"is":[59],"critical":[60],"establish":[62],"common":[64],"ground":[65],"quality":[67,96],"assessment":[68],"results.":[71],"paper":[74,167],"we":[75],"report":[76],"our":[77,130],"experiences":[78],"optimizing":[80],"adapting":[82],"microtasks":[84],"considering":[87],"three":[88],"aspects:":[89],"(1)":[90],"micro-task":[92],"template,":[93],"(2)":[94],"measurements":[97],"workers":[100],"judgments":[101],"(3)":[103],"overall":[105],"work":[107],"ow.":[108],"We":[109],"performed":[110],"experiments":[111],"in":[112,144],"two":[113],"domains,":[114],"i.e.":[115],"events":[116],"extraction":[117,123,178],"(MRP":[118],"project)":[119],"medical":[121,189],"relations":[122],"(Crowd-Watson":[124],"project).":[125],"The":[126],"results":[127],"con":[128],"rm":[129],"main":[131],"hypothesis":[132],"that":[133,150],"some":[134],"aspects":[135],"evaluation":[138],"metrics":[139],"de":[142],"ned":[143],"domainindependent":[146],"way":[147],"micro-tasks":[149],"assess":[151],"parameters":[153,173],"annotations":[159],"useful":[162],"disagreement":[163],"between":[164],"workers.":[165],"This":[166],"focuses":[168],"speci":[169],"cally":[170],"on":[171],"relevant":[174],"'event":[177],"'":[179],"ground-truth":[180],"collection":[182],"demonstrates":[184],"their":[185],"reusability":[186],"from":[187],"domain.":[190]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2899357352","counts_by_year":[{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":2},{"year":2016,"cited_by_count":4},{"year":2014,"cited_by_count":1},{"year":2013,"cited_by_count":1}],"updated_date":"2026-11-21T10:27:49.728292","created_date":"2018-11-09T00:00:00"}
