{"id":"https://openalex.org/W3046656635","doi":"https://doi.org/10.1145/3383583.3398537","title":"Seeking Justification: How Expert Reviewers Validate Empirical Claims with Data Annotations","display_name":"Seeking Justification: How Expert Reviewers Validate Empirical Claims with Data Annotations","publication_year":2020,"publication_date":"2020-08-01","ids":{"openalex":"https://openalex.org/W3046656635","doi":"https://doi.org/10.1145/3383583.3398537","mag":"3046656635"},"language":"en","primary_location":{"id":"doi:10.1145/3383583.3398537","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3383583.3398537","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM/IEEE Joint Conference on Digital Libraries in 2020","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085874183","display_name":"Nicholas Weber","orcid":"https://orcid.org/0000-0002-6008-3763"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nicholas Weber","raw_affiliation_strings":["University of Washington, Seattle, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066039394","display_name":"Sebastian Karcher","orcid":"https://orcid.org/0000-0001-8249-7388"},"institutions":[{"id":"https://openalex.org/I70983195","display_name":"Syracuse University","ror":"https://ror.org/025r5qe02","country_code":"US","type":"education","lineage":["https://openalex.org/I70983195"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sebastian Karcher","raw_affiliation_strings":["Syracuse University, Syracuse, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Syracuse University, Syracuse, NY, USA","institution_ids":["https://openalex.org/I70983195"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.9102,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.86619291,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"227","last_page":"234"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9950000047683716,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11937","display_name":"Research Data Management Practices","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10102","display_name":"scientometrics and bibliometrics research","score":0.9864000082015991,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transparency","display_name":"Transparency (behavior)","score":0.7878738045692444},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7172442078590393},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5673468112945557},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.5631248354911804},{"id":"https://openalex.org/keywords/data-sharing","display_name":"Data sharing","score":0.5510174632072449},{"id":"https://openalex.org/keywords/peer-production","display_name":"Peer production","score":0.47471630573272705},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.4264358878135681},{"id":"https://openalex.org/keywords/data-access","display_name":"Data access","score":0.41605156660079956},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3790602684020996},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.11542192101478577},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.10496005415916443}],"concepts":[{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.7878738045692444},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7172442078590393},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5673468112945557},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.5631248354911804},{"id":"https://openalex.org/C2779965156","wikidata":"https://www.wikidata.org/wiki/Q5227350","display_name":"Data sharing","level":3,"score":0.5510174632072449},{"id":"https://openalex.org/C2779507501","wikidata":"https://www.wikidata.org/wiki/Q797769","display_name":"Peer production","level":2,"score":0.47471630573272705},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.4264358878135681},{"id":"https://openalex.org/C47487241","wikidata":"https://www.wikidata.org/wiki/Q5227230","display_name":"Data access","level":2,"score":0.41605156660079956},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3790602684020996},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.11542192101478577},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.10496005415916443},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C204787440","wikidata":"https://www.wikidata.org/wiki/Q188504","display_name":"Alternative medicine","level":2,"score":0.0},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3383583.3398537","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3383583.3398537","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM/IEEE Joint Conference on Digital Libraries in 2020","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G8002937360","display_name":null,"funder_award_id":"74422","funder_id":"https://openalex.org/F4320306139","funder_display_name":"Robert Wood Johnson Foundation"}],"funders":[{"id":"https://openalex.org/F4320306139","display_name":"Robert Wood Johnson Foundation","ror":"https://ror.org/02ymmdj85"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W1575893985","https://openalex.org/W1909985944","https://openalex.org/W1969790497","https://openalex.org/W1975004953","https://openalex.org/W1986767108","https://openalex.org/W2004776253","https://openalex.org/W2012004704","https://openalex.org/W2025891673","https://openalex.org/W2029099961","https://openalex.org/W2046766973","https://openalex.org/W2077618845","https://openalex.org/W2077746342","https://openalex.org/W2109642736","https://openalex.org/W2115720312","https://openalex.org/W2128401058","https://openalex.org/W2135572748","https://openalex.org/W2235693693","https://openalex.org/W2302028439","https://openalex.org/W2313851908","https://openalex.org/W2336977259","https://openalex.org/W2560961381","https://openalex.org/W2582635907","https://openalex.org/W2737578613","https://openalex.org/W2782278703","https://openalex.org/W2783849151","https://openalex.org/W2803067141","https://openalex.org/W2809346643","https://openalex.org/W2908654177","https://openalex.org/W2954847854","https://openalex.org/W2996385440","https://openalex.org/W3000457005","https://openalex.org/W4247533034","https://openalex.org/W6906235124"],"related_works":["https://openalex.org/W4382930947","https://openalex.org/W3081288631","https://openalex.org/W3152382318","https://openalex.org/W3004686567","https://openalex.org/W2414507368","https://openalex.org/W2166798126","https://openalex.org/W4389950265","https://openalex.org/W2421443948","https://openalex.org/W2050921398","https://openalex.org/W4226034287"],"abstract_inverted_index":{"We":[0,17,115],"investigate":[1],"the":[2,34,158],"use":[3],"of":[4,36,66,113,160],"data":[5,11,43,67,147],"annotations":[6,49],"to":[7,27,53,140,145],"increase":[8],"transparency":[9],"and":[10,32,44,109,126,156],"access":[12,52],"in":[13,58,86,130,137],"social":[14],"science":[15],"research.":[16],"present":[18],"preliminary":[19],"findings":[20,119],"from":[21],"a":[22,29],"study":[23],"that":[24,50,57,93,100,117],"asked":[25],"experts":[26],"re-view":[28],"research":[30],"paper":[31],"judge":[33],"validity":[35],"empirical":[37],"claims":[38],"made":[39],"-":[40],"first":[41],"without":[42],"then":[45],"with":[46,152],"author":[47],"supplied":[48],"provide":[51],"underlying":[54],"data.We":[55],"demonstrate":[56],"domains":[59],"where":[60],"there":[61],"is":[62],"an":[63,87],"emergent":[64],"culture":[65],"sharing":[68],"reviewer":[69],"expectations":[70],"about":[71],"what":[72],"material":[73],"should":[74],"be":[75],"shared":[76,94],"are":[77],"met,":[78],"but":[79,99],"this":[80],"does":[81],"not":[82],"necessarily":[83],"influence":[84],"trust":[85],"author's":[88],"claims.":[89],"Reviewers":[90],"describe":[91],"ways":[92],"materials":[95,102],"aide":[96],"validation":[97],"tasks,":[98],"these":[101,118,133],"just":[103],"as":[104,132],"often":[105],"sharpen":[106],"their":[107],"critique":[108],"introduce":[110],"new":[111],"lines":[112],"questioning.":[114],"believe":[116],"hold":[120],"important":[121],"implications":[122],"for":[123,148,157],"scholarly":[124],"communications":[125],"digital":[127],"repository":[128,154],"development,":[129],"particular":[131],"two":[134],"communities":[135],"engage":[136],"developing":[138],"policies":[139],"guide":[141],"authors":[142],"on":[143],"how":[144],"prepare":[146],"review,":[149],"supporting":[150],"reviewers":[151],"linked":[153],"systems,":[155],"design":[159],"peer-review":[161],"processes":[162],"more":[163],"generally.":[164]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}