{"id":"https://openalex.org/W2142119621","doi":"https://doi.org/10.1145/2702123.2702223","title":"Formalizing Agreement Analysis for Elicitation Studies","display_name":"Formalizing Agreement Analysis for Elicitation Studies","publication_year":2015,"publication_date":"2015-04-17","ids":{"openalex":"https://openalex.org/W2142119621","doi":"https://doi.org/10.1145/2702123.2702223","mag":"2142119621"},"language":"en","primary_location":{"id":"doi:10.1145/2702123.2702223","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2702123.2702223","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033726877","display_name":"Radu-Daniel Vatavu","orcid":"https://orcid.org/0000-0002-7631-6445"},"institutions":[{"id":"https://openalex.org/I12012874","display_name":"\u015etefan cel Mare University of Suceava","ror":"https://ror.org/035pkj773","country_code":"RO","type":"education","lineage":["https://openalex.org/I12012874"]}],"countries":["RO"],"is_corresponding":false,"raw_author_name":"Radu-Daniel Vatavu","raw_affiliation_strings":["University Stefan cel Mare of Suceava, Suceava, Romania","[University Stefan cel Mare of Suceava, Suceava, Romania]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University Stefan cel Mare of Suceava, Suceava, Romania","institution_ids":["https://openalex.org/I12012874"]},{"raw_affiliation_string":"[University Stefan cel Mare of Suceava, Suceava, Romania]","institution_ids":["https://openalex.org/I12012874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5086377685","display_name":"Jacob O. Wobbrock","orcid":"https://orcid.org/0000-0003-3675-5491"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jacob O. Wobbrock","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":233,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1325","last_page":"1334"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10914","display_name":"Tactile and Sensory Interactions","score":0.9851999878883362,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9817000031471252,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/analogy","display_name":"Analogy","score":0.7501710057258606},{"id":"https://openalex.org/keywords/statistic","display_name":"Statistic","score":0.6082343459129333},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.5746234655380249},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.531866192817688},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5170722603797913},{"id":"https://openalex.org/keywords/test-statistic","display_name":"Test statistic","score":0.4822038412094116},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.4811364710330963},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.414595365524292},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.41091763973236084},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3366340398788452},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.3334695100784302},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.15943056344985962},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.12648001313209534}],"concepts":[{"id":"https://openalex.org/C521332185","wikidata":"https://www.wikidata.org/wiki/Q185816","display_name":"Analogy","level":2,"score":0.7501710057258606},{"id":"https://openalex.org/C89128539","wikidata":"https://www.wikidata.org/wiki/Q1949963","display_name":"Statistic","level":2,"score":0.6082343459129333},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.5746234655380249},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.531866192817688},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5170722603797913},{"id":"https://openalex.org/C169857963","wikidata":"https://www.wikidata.org/wiki/Q1461038","display_name":"Test statistic","level":3,"score":0.4822038412094116},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4811364710330963},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.414595365524292},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.41091763973236084},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3366340398788452},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3334695100784302},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.15943056344985962},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.12648001313209534},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2702123.2702223","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2702123.2702223","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.41999998688697815,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W119458596","https://openalex.org/W177399310","https://openalex.org/W1545424562","https://openalex.org/W1971876305","https://openalex.org/W1978580638","https://openalex.org/W1985450022","https://openalex.org/W1996349741","https://openalex.org/W2004986590","https://openalex.org/W2015776973","https://openalex.org/W2021264779","https://openalex.org/W2025647423","https://openalex.org/W2026519996","https://openalex.org/W2029454602","https://openalex.org/W2033228500","https://openalex.org/W2033955170","https://openalex.org/W2035487231","https://openalex.org/W2037557484","https://openalex.org/W2045260557","https://openalex.org/W2050034390","https://openalex.org/W2053154970","https://openalex.org/W2056433986","https://openalex.org/W2065228210","https://openalex.org/W2083370515","https://openalex.org/W2084413241","https://openalex.org/W2086449672","https://openalex.org/W2099372034","https://openalex.org/W2100697272","https://openalex.org/W2119593195","https://openalex.org/W2125802131","https://openalex.org/W2127707768","https://openalex.org/W2128213565","https://openalex.org/W2143558937","https://openalex.org/W2154876275","https://openalex.org/W2157134953","https://openalex.org/W2157771033","https://openalex.org/W2161304134","https://openalex.org/W2163724930","https://openalex.org/W2495580237","https://openalex.org/W2997610230","https://openalex.org/W4214490056","https://openalex.org/W4251081571"],"related_works":["https://openalex.org/W3015822731","https://openalex.org/W2957280400","https://openalex.org/W2075332338","https://openalex.org/W1970221037","https://openalex.org/W4386776152","https://openalex.org/W1826709224","https://openalex.org/W2135903028","https://openalex.org/W2488034322","https://openalex.org/W2913739054","https://openalex.org/W4385820860"],"abstract_inverted_index":{"We":[0,94,123,189,222],"address":[1],"in":[2,72,87,132,168,180,232],"this":[3,220],"work":[4],"the":[5,13,41,64,84,145,150,161,169,197,200,210,229],"process":[6],"of":[7,15,46,67,89,121,128,149,163,199,212,254],"agreement":[8,43,65,90,113,129,151,164,178,201,215,233],"rate":[9,29,34,44,130,152,202,234],"analysis":[10,131],"for":[11,30,55,62,80,112,177,185],"characterizing":[12],"level":[14],"consensus":[16,54],"between":[17,35],"participants'":[18,53],"proposals":[19],"elicited":[20],"during":[21],"guessability":[22],"studies.":[23,139],"Two":[24],"new":[25,207,241],"measures,":[26],"i.e.,":[27],"disagreement":[28],"referents":[31,69,213],"and":[32,91,105,108,118,171,205,238,243,256],"coagreement":[33,92,106],"referents,":[36],"are":[37],"proposed":[38],"to":[39,98,101,159,173,219,245],"accompany":[40],"widely-used":[42],"formula":[45],"Wobbrock":[47],"et":[48],"al.":[49],"[37]":[50],"when":[51,142],"reporting":[52],"symbolic":[56],"input.":[57],"A":[58],"statistical":[59,110],"significance":[60],"test":[61,78,85,203],"comparing":[63],"rates":[66,114,165],"k>=2":[68],"is":[70],"presented":[71],"analogy":[73,181],"with":[74,134,182,240],"Cochran's":[75],"success/failure":[76],"Q":[77],"[5],":[79],"which":[81],"we":[82,143,154],"express":[83],"statistic":[86],"terms":[88],"rates.":[93],"deliver":[95],"a":[96],"toolkit":[97],"assist":[99],"practitioners":[100,239],"compute":[102],"agreement,":[103],"disagreement,":[104],"rates,":[107,179,216],"run":[109],"tests":[111],"at":[115,251],"p=.05,":[116],".01,":[117],".001":[119],"levels":[120,253],"significance.":[122],"validate":[124],"our":[125,225],"theoretical":[126],"development":[127],"relation":[133],"several":[135],"previously":[136,166,192],"published":[137,193],"elicitation":[138,194],"For":[140],"example,":[141],"present":[144],"probability":[146],"distribution":[147],"function":[148],"measure,":[153],"also":[155,190],"use":[156],"it":[157],"(1)":[158],"explain":[160],"magnitude":[162],"reported":[167],"literature,":[170],"(2)":[172],"propose":[174],"qualitative":[175],"interpretations":[176],"Cohen's":[183],"guidelines":[184],"effect":[186,211],"sizes":[187],"[6].":[188],"re-examine":[191],"data":[195,250],"from":[196],"perspective":[198],"statistic,":[204],"highlight":[206],"findings":[208],"on":[209],"over":[214],"unattainable":[217],"prior":[218],"work.":[221],"hope":[223],"that":[224],"contributions":[226],"will":[227],"advance":[228],"current":[230],"knowledge":[231],"analysis,":[235],"providing":[236],"researchers":[237],"techniques":[242],"tools":[244],"help":[246],"them":[247],"understand":[248],"user-elicited":[249],"deeper":[252],"detail":[255],"sophistication.":[257]},"counts_by_year":[{"year":2026,"cited_by_count":12},{"year":2025,"cited_by_count":23},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":17},{"year":2022,"cited_by_count":31},{"year":2021,"cited_by_count":25},{"year":2020,"cited_by_count":32},{"year":2019,"cited_by_count":32},{"year":2018,"cited_by_count":20},{"year":2017,"cited_by_count":16},{"year":2016,"cited_by_count":9},{"year":2015,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}