{"id":"https://openalex.org/W2473103291","doi":"https://doi.org/10.1108/ajim-10-2015-0165","title":"Testing the stability of \u201cwisdom of crowds\u201d judgments of search results over time and their similarity with the search engine rankings","display_name":"Testing the stability of \u201cwisdom of crowds\u201d judgments of search results over time and their similarity with the search engine rankings","publication_year":2016,"publication_date":"2016-06-30","ids":{"openalex":"https://openalex.org/W2473103291","doi":"https://doi.org/10.1108/ajim-10-2015-0165","mag":"2473103291"},"language":"en","primary_location":{"id":"doi:10.1108/ajim-10-2015-0165","is_oa":false,"landing_page_url":"https://doi.org/10.1108/ajim-10-2015-0165","pdf_url":null,"source":{"id":"https://openalex.org/S4210181081","display_name":"Aslib Journal of Information Management","issn_l":"2050-3806","issn":["2050-3806","2050-3814"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319811","host_organization_name":"Emerald Publishing Limited","host_organization_lineage":["https://openalex.org/P4310319811"],"host_organization_lineage_names":["Emerald Publishing Limited"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Aslib Journal of Information Management","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006766510","display_name":"Maayan Zhitomirsky\u2010Geffet","orcid":"https://orcid.org/0000-0002-9161-2541"},"institutions":[{"id":"https://openalex.org/I13955877","display_name":"Bar-Ilan University","ror":"https://ror.org/03kgsv495","country_code":"IL","type":"education","lineage":["https://openalex.org/I13955877"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Maayan Zhitomirsky-Geffet","raw_affiliation_strings":["Department of Information Science,Bar-Ilan University,Ramat Gan,Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Information Science,Bar-Ilan University,Ramat Gan,Israel","institution_ids":["https://openalex.org/I13955877"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001893735","display_name":"Judit Bar\u2010Ilan","orcid":"https://orcid.org/0000-0002-8796-5248"},"institutions":[{"id":"https://openalex.org/I13955877","display_name":"Bar-Ilan University","ror":"https://ror.org/03kgsv495","country_code":"IL","type":"education","lineage":["https://openalex.org/I13955877"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Judit Bar-Ilan","raw_affiliation_strings":["Department of Information Science,Bar-Ilan University,Ramat Gan,Israel"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Information Science,Bar-Ilan University,Ramat Gan,Israel","institution_ids":["https://openalex.org/I13955877"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015725705","display_name":"Mark Levene","orcid":"https://orcid.org/0000-0001-8632-4732"},"institutions":[{"id":"https://openalex.org/I98259816","display_name":"Birkbeck, University of London","ror":"https://ror.org/02mb95055","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I98259816"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mark Levene","raw_affiliation_strings":["Department of Computer Science,BirkBeck University of London,London,UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science,BirkBeck University of London,London,UK","institution_ids":["https://openalex.org/I98259816"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.7614,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.92034562,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"68","issue":"4","first_page":"407","last_page":"427"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7516428232192993},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.7252873778343201},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.647468090057373},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6133444309234619},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.6066625118255615},{"id":"https://openalex.org/keywords/search-engine","display_name":"Search engine","score":0.5619288086891174},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.560844898223877},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.553724467754364},{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.526852011680603},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.47748658061027527},{"id":"https://openalex.org/keywords/personalization","display_name":"Personalization","score":0.47002530097961426},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4216544032096863},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.3652191758155823},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.20631125569343567},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.1699603796005249},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1544056236743927},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08864384889602661}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7516428232192993},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7252873778343201},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.647468090057373},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6133444309234619},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.6066625118255615},{"id":"https://openalex.org/C97854310","wikidata":"https://www.wikidata.org/wiki/Q19541","display_name":"Search engine","level":2,"score":0.5619288086891174},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.560844898223877},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.553724467754364},{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.526852011680603},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.47748658061027527},{"id":"https://openalex.org/C183003079","wikidata":"https://www.wikidata.org/wiki/Q1000371","display_name":"Personalization","level":2,"score":0.47002530097961426},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4216544032096863},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3652191758155823},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.20631125569343567},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.1699603796005249},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1544056236743927},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08864384889602661},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1108/ajim-10-2015-0165","is_oa":false,"landing_page_url":"https://doi.org/10.1108/ajim-10-2015-0165","pdf_url":null,"source":{"id":"https://openalex.org/S4210181081","display_name":"Aslib Journal of Information Management","issn_l":"2050-3806","issn":["2050-3806","2050-3814"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319811","host_organization_name":"Emerald Publishing Limited","host_organization_lineage":["https://openalex.org/P4310319811"],"host_organization_lineage_names":["Emerald Publishing Limited"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Aslib Journal of Information Management","raw_type":"journal-article"},{"id":"pmh:oai:eprints.bbk.ac.uk.oai2:14848","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306400466","display_name":"BIROn (Birkbeck, University of London)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98259816","host_organization_name":"Birkbeck, University of London","host_organization_lineage":["https://openalex.org/I98259816"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":60,"referenced_works":["https://openalex.org/W184460951","https://openalex.org/W319424671","https://openalex.org/W1492458989","https://openalex.org/W1500609524","https://openalex.org/W1502214077","https://openalex.org/W1537525764","https://openalex.org/W1541372434","https://openalex.org/W1552448790","https://openalex.org/W1561661463","https://openalex.org/W1802760973","https://openalex.org/W1833752439","https://openalex.org/W1964729842","https://openalex.org/W1964817118","https://openalex.org/W1966197090","https://openalex.org/W1966903812","https://openalex.org/W1972644898","https://openalex.org/W1973900256","https://openalex.org/W1974360117","https://openalex.org/W1979464183","https://openalex.org/W1981745473","https://openalex.org/W1982149803","https://openalex.org/W1996848687","https://openalex.org/W2010210047","https://openalex.org/W2020713996","https://openalex.org/W2027860007","https://openalex.org/W2045028206","https://openalex.org/W2051497960","https://openalex.org/W2055373607","https://openalex.org/W2056085308","https://openalex.org/W2059873455","https://openalex.org/W2065108642","https://openalex.org/W2069200549","https://openalex.org/W2071446250","https://openalex.org/W2073338313","https://openalex.org/W2089873046","https://openalex.org/W2094250098","https://openalex.org/W2094607109","https://openalex.org/W2096748234","https://openalex.org/W2097193844","https://openalex.org/W2101618658","https://openalex.org/W2110961693","https://openalex.org/W2117285976","https://openalex.org/W2125771191","https://openalex.org/W2131508194","https://openalex.org/W2135645798","https://openalex.org/W2138918086","https://openalex.org/W2139893373","https://openalex.org/W2145734075","https://openalex.org/W2147709600","https://openalex.org/W2149202473","https://openalex.org/W2155355283","https://openalex.org/W2337312799","https://openalex.org/W2402561932","https://openalex.org/W2403095230","https://openalex.org/W2913081710","https://openalex.org/W4206671592","https://openalex.org/W4254168416","https://openalex.org/W4255703762","https://openalex.org/W4285719527","https://openalex.org/W4300831780"],"related_works":["https://openalex.org/W4240200267","https://openalex.org/W1511510665","https://openalex.org/W1524661185","https://openalex.org/W2078823605","https://openalex.org/W2500095415","https://openalex.org/W4233026749","https://openalex.org/W2097922264","https://openalex.org/W2282342021","https://openalex.org/W2367099342","https://openalex.org/W2116655434"],"abstract_inverted_index":{"Purpose":[0],"\u2013":[1,94,197,223,264],"One":[2],"of":[3,10,17,36,48,66,70,108,126,141,171,176,227,238,249,274,276,286,293],"the":[4,8,64,105,123,136,236,247,272,283],"under-explored":[5,281],"aspects":[6],"in":[7,26,180,260,282,288,290],"process":[9],"user":[11,91,119,150,210,291],"information":[12],"seeking":[13],"behaviour":[14],"is":[15,42,61,73,242,266],"influence":[16],"time":[18,88,257],"on":[19,184,190],"relevance":[20,183],"evaluation.":[21],"It":[22,41,198],"has":[23],"been":[24],"shown":[25],"previous":[27],"studies":[28],"that":[29,45,201,270],"individual":[30,50,90,118,209],"users":[31,51,261],"might":[32],"change":[33],"their":[34,113],"assessment":[35],"search":[37,109,142,145,218,239,253],"results":[38,110,143],"over":[39,87,256],"time.":[40],"also":[43,243],"known":[44,62],"aggregated":[46,77,106,124,202],"judgements":[47,78,107,128,177,203],"multiple":[49],"can":[52],"lead":[53],"to":[54,74,103,117,134,258,278],"correct":[55],"and":[56,83,111,115,144,162,188,251],"reliable":[57,86,232],"decisions;":[58],"this":[59,71,96,181],"phenomenon":[60,285],"as":[63,131],"\u201cwisdom":[65,125,226,275],"crowds\u201d.":[67],"The":[68,224],"purpose":[69],"paper":[72],"examine":[75,279],"whether":[76],"will":[79],"be":[80],"more":[81,85,206],"stable":[82,207],"thus":[84],"than":[89,208],"judgements.":[92,120],"Design/methodology/approach":[93],"In":[95,121],"study":[97,151,269],"two":[98,159,172],"simple":[99],"measures":[100],"are":[101,204,214],"proposed":[102,225],"calculate":[104],"compare":[112,135],"reliability":[114],"stability":[116],"addition,":[122],"crowds\u201d":[127,277],"were":[129,178],"used":[130],"a":[132,185,191,231,267],"means":[133],"differences":[137],"between":[138],"human":[139],"assessments":[140],"engine\u2019s":[146,254],"rankings.":[147,220],"A":[148],"large-scale":[149],"was":[152,199],"conducted":[153],"with":[154,168],"87":[155],"participants":[156],"who":[157],"evaluated":[158],"different":[160,216],"queries":[161],"four":[163],"diverse":[164],"result":[165],"sets":[166],"twice,":[167],"an":[169,280],"interval":[170],"months.":[173],"Two":[174],"types":[175],"considered":[179],"study:":[182],"four-point":[186],"scale,":[187],"ranking":[189,255],"ten-point":[192],"scale":[193],"without":[194],"ties.":[195],"Findings":[196],"found":[200],"much":[205],"judgements,":[211],"yet":[212],"they":[213],"quite":[215],"from":[217],"engine":[219],"Practical":[221],"implications":[222],"crowds\u201d-based":[228],"approach":[229],"provides":[230],"reference":[233],"point":[234],"for":[235,245],"evaluation":[237,292],"engines.":[240],"This":[241,265],"important":[244],"exploring":[246],"need":[248],"personalisation":[250],"adapting":[252],"changes":[259],"preferences.":[262],"Originality/value":[263],"first":[268],"applies":[271],"notion":[273],"literature":[284],"\u201cchange":[287],"time\u201d":[289],"relevance.":[294]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}