{"id":"https://openalex.org/W2071446250","doi":"https://doi.org/10.1145/2484028.2484090","title":"The effect of threshold priming and need for cognition on relevance calibration and assessment","display_name":"The effect of threshold priming and need for cognition on relevance calibration and assessment","publication_year":2013,"publication_date":"2013-07-28","ids":{"openalex":"https://openalex.org/W2071446250","doi":"https://doi.org/10.1145/2484028.2484090","mag":"2071446250"},"language":"en","primary_location":{"id":"doi:10.1145/2484028.2484090","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2484028.2484090","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 36th international ACM SIGIR conference on Research and development in information retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://figshare.com/articles/conference_contribution/The_effect_of_threshold_priming_and_need_for_cognition_on_relevance_calibration_and_assessment/27384216","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069419532","display_name":"Diane Kelly","orcid":"https://orcid.org/0000-0003-2231-0192"},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Diane Kelly","raw_affiliation_strings":["University of North Carolina, Chapel Hill, NC, USA"],"affiliations":[{"raw_affiliation_string":"University of North Carolina, Chapel Hill, NC, USA","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021633850","display_name":"Wan\u2010Ching Wu","orcid":null},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wan-Ching Wu","raw_affiliation_strings":["University of North Carolina, Chapel Hill, NC, USA"],"affiliations":[{"raw_affiliation_string":"University of North Carolina, Chapel Hill, NC, USA","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086443774","display_name":"Hanseul S. Lee","orcid":null},"institutions":[{"id":"https://openalex.org/I114027177","display_name":"University of North Carolina at Chapel Hill","ror":"https://ror.org/0130frc33","country_code":"US","type":"education","lineage":["https://openalex.org/I114027177"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hanseul S. Lee","raw_affiliation_strings":["University of North Carolina, Chapel Hill, NC, USA"],"affiliations":[{"raw_affiliation_string":"University of North Carolina, Chapel Hill, NC, USA","institution_ids":["https://openalex.org/I114027177"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5110722281","display_name":"William Webber","orcid":null},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"William Webber","raw_affiliation_strings":["University of Maryland, College Park, MD, USA"],"affiliations":[{"raw_affiliation_string":"University of Maryland, College Park, MD, USA","institution_ids":["https://openalex.org/I66946132"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5085925326"],"corresponding_institution_ids":["https://openalex.org/I82951845"],"apc_list":null,"apc_paid":null,"fwci":11.3674,"has_fulltext":false,"cited_by_count":72,"citation_normalized_percentile":{"value":0.98293068,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"623","last_page":"632"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12887","display_name":"Educational Strategies and Epistemologies","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8872090578079224},{"id":"https://openalex.org/keywords/prologue","display_name":"Prologue","score":0.821577787399292},{"id":"https://openalex.org/keywords/cognition","display_name":"Cognition","score":0.7393125295639038},{"id":"https://openalex.org/keywords/priming","display_name":"Priming (agriculture)","score":0.5736282467842102},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.526788055896759},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.45113521814346313},{"id":"https://openalex.org/keywords/need-for-cognition","display_name":"Need for cognition","score":0.4414634704589844},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.39598560333251953},{"id":"https://openalex.org/keywords/history","display_name":"History","score":0.07683518528938293}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8872090578079224},{"id":"https://openalex.org/C2780627709","wikidata":"https://www.wikidata.org/wiki/Q920285","display_name":"Prologue","level":2,"score":0.821577787399292},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.7393125295639038},{"id":"https://openalex.org/C81444415","wikidata":"https://www.wikidata.org/wiki/Q7243535","display_name":"Priming (agriculture)","level":3,"score":0.5736282467842102},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.526788055896759},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.45113521814346313},{"id":"https://openalex.org/C53371666","wikidata":"https://www.wikidata.org/wiki/Q1778408","display_name":"Need for cognition","level":3,"score":0.4414634704589844},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.39598560333251953},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.07683518528938293},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C100701293","wikidata":"https://www.wikidata.org/wiki/Q193838","display_name":"Germination","level":2,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1145/2484028.2484090","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2484028.2484090","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 36th international ACM SIGIR conference on Research and development in information retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:researchbank.rmit.edu.au:rmit:22920","is_oa":false,"landing_page_url":"http://researchbank.rmit.edu.au/view/rmit:22920","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Paper"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.352.9759","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.352.9759","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.umiacs.umd.edu/~wew/papers/skwlw13sigir.pdf","raw_type":"text"},{"id":"pmh:oai:alma.61RMIT_INST:11246918660001341","is_oa":false,"landing_page_url":"http://doi.org/10.1145/2484028.2484090","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"pmh:oai:figshare.com:article/27384216","is_oa":true,"landing_page_url":"https://figshare.com/articles/conference_contribution/The_effect_of_threshold_priming_and_need_for_cognition_on_relevance_calibration_and_assessment/27384216","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:figshare.com:article/27384216","is_oa":true,"landing_page_url":"https://figshare.com/articles/conference_contribution/The_effect_of_threshold_priming_and_need_for_cognition_on_relevance_calibration_and_assessment/27384216","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.699999988079071}],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320334704","display_name":"Australian Research Council","ror":"https://ror.org/05mmh0f86"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":36,"referenced_works":["https://openalex.org/W133870725","https://openalex.org/W340682576","https://openalex.org/W396175112","https://openalex.org/W1561661463","https://openalex.org/W1963800712","https://openalex.org/W1965995226","https://openalex.org/W1969587718","https://openalex.org/W1974548087","https://openalex.org/W1979483312","https://openalex.org/W1981065051","https://openalex.org/W1982149803","https://openalex.org/W1982343866","https://openalex.org/W1991774506","https://openalex.org/W2003371341","https://openalex.org/W2017856924","https://openalex.org/W2023292494","https://openalex.org/W2029192830","https://openalex.org/W2044361482","https://openalex.org/W2048987619","https://openalex.org/W2053397947","https://openalex.org/W2053528567","https://openalex.org/W2062236692","https://openalex.org/W2070204737","https://openalex.org/W2092488901","https://openalex.org/W2094607109","https://openalex.org/W2110420103","https://openalex.org/W2120308175","https://openalex.org/W2121044470","https://openalex.org/W2121821044","https://openalex.org/W2141062075","https://openalex.org/W2141653139","https://openalex.org/W2141902457","https://openalex.org/W2153763032","https://openalex.org/W2155961949","https://openalex.org/W2187291759","https://openalex.org/W2329344288"],"related_works":["https://openalex.org/W2020604165","https://openalex.org/W2031681959","https://openalex.org/W2078614653","https://openalex.org/W4403429458","https://openalex.org/W2972731406","https://openalex.org/W2529876606","https://openalex.org/W2089184728","https://openalex.org/W2074194212","https://openalex.org/W1975988730","https://openalex.org/W4389410911"],"abstract_inverted_index":{"Human":[0],"assessments":[1],"of":[2,10,45,52,58,61,76,78,127,153],"document":[3],"relevance":[4,57,97,141,177,189],"are":[5],"needed":[6],"for":[7,13,17,121,145,161],"the":[8,37,56,91,114,128,181],"construction":[9],"test":[11],"collections,":[12],"ad-hoc":[14],"evaluation,":[15],"and":[16,101,195],"training":[18],"text":[19],"classifiers.":[20],"Showing":[21],"documents":[22,62,77,89,103,112,174],"to":[23,31,86,99,107,130,173,186,205],"assessors":[24,157,169],"in":[25,90,113,136,180,184,191],"different":[26,32],"orderings,":[27],"however,":[28],"may":[29],"lead":[30],"assessment":[33],"outcomes.":[34],"We":[35,81,116],"examine":[36,118],"effect":[38],"that":[39,83,168,196],"\\defineterm{threshold":[40],"priming},":[41],"seeing":[42],"varying":[43],"degrees":[44],"relevant":[46,111],"documents,":[47],"has":[48],"on":[49],"people's":[50],"calibration":[51],"relevance.":[53,80],"Participants":[54],"judged":[55],"a":[59,73,132,149,192,202],"prologue":[60,92,100],"containing":[63],"highly":[64,110],"relevant,":[65,67],"moderately":[66,108],"or":[68,109],"non-relevant":[69,88],"ocuments,":[70],"followed":[71],"by":[72],"common":[74],"epilogue":[75,102],"mixed":[79],"observe":[82],"participants":[84,105,147,163],"exposed":[85,106,172],"only":[87],"assigned":[93],"significantly":[94,150],"higher":[95,151],"average":[96],"scores":[98],"than":[104,158],"prologue.":[115],"also":[117],"how":[119],"\\defineterm{need":[120],"cognition},":[122],"an":[123],"individual":[124,197],"difference":[125,198],"measure":[126],"extent":[129],"which":[131],"person":[133],"enjoys":[134],"engaging":[135],"effortful":[137],"cognitive":[138],"activity,":[139],"impacts":[140],"assessments.":[142],"High":[143],"need":[144,160],"cognition":[146,162],"had":[148],"level":[152],"agreement":[154],"with":[155],"expert":[156],"low":[159],"did.":[164],"Our":[165],"findings":[166],"indicate":[167],"should":[170],"be":[171,201],"from":[175],"multiple":[176],"levels":[178],"early":[179],"judging":[182],"process,":[183],"order":[185],"calibrate":[187],"their":[188],"thresholds":[190],"balanced":[193],"way,":[194],"measures":[199],"might":[200],"useful":[203],"way":[204],"screen":[206],"assessors.":[207]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":11},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":9},{"year":2015,"cited_by_count":8},{"year":2014,"cited_by_count":6}],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2025-10-10T00:00:00"}