{"id":"https://openalex.org/W2896758679","doi":"https://doi.org/10.1108/ajim-04-2018-0086","title":"Evaluating the effectiveness of information retrieval systems using effort-based relevance judgment","display_name":"Evaluating the effectiveness of information retrieval systems using effort-based relevance judgment","publication_year":2018,"publication_date":"2018-10-11","ids":{"openalex":"https://openalex.org/W2896758679","doi":"https://doi.org/10.1108/ajim-04-2018-0086","mag":"2896758679"},"language":"en","primary_location":{"id":"doi:10.1108/ajim-04-2018-0086","is_oa":false,"landing_page_url":"https://doi.org/10.1108/ajim-04-2018-0086","pdf_url":null,"source":{"id":"https://openalex.org/S4210189668","display_name":"Aslib Journal of Information Management","issn_l":"0001-253X","issn":["0001-253X","1758-3748","2050-3806","2050-3814"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319811","host_organization_name":"Emerald Publishing Limited","host_organization_lineage":["https://openalex.org/P4310319811"],"host_organization_lineage_names":["Emerald Publishing Limited"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Aslib Journal of Information Management","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081983920","display_name":"Prabha Rajagopal","orcid":"https://orcid.org/0000-0002-5734-3691"},"institutions":[{"id":"https://openalex.org/I33849332","display_name":"University of Malaya","ror":"https://ror.org/00rzspn62","country_code":"MY","type":"education","lineage":["https://openalex.org/I33849332"]}],"countries":["MY"],"is_corresponding":false,"raw_author_name":"Prabha Rajagopal","raw_affiliation_strings":["University of Malaya, Kuala Lumpur, Malaysia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Malaya, Kuala Lumpur, Malaysia","institution_ids":["https://openalex.org/I33849332"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007931408","display_name":"Sri Devi Ravana","orcid":"https://orcid.org/0000-0002-5637-9158"},"institutions":[{"id":"https://openalex.org/I33849332","display_name":"University of Malaya","ror":"https://ror.org/00rzspn62","country_code":"MY","type":"education","lineage":["https://openalex.org/I33849332"]}],"countries":["MY"],"is_corresponding":false,"raw_author_name":"Sri Devi Ravana","raw_affiliation_strings":["University of Malaya, Kuala Lumpur, Malaysia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Malaya, Kuala Lumpur, Malaysia","institution_ids":["https://openalex.org/I33849332"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017570709","display_name":"Yun Sing Koh","orcid":"https://orcid.org/0000-0001-7256-4049"},"institutions":[{"id":"https://openalex.org/I154130895","display_name":"University of Auckland","ror":"https://ror.org/03b94tp07","country_code":"NZ","type":"education","lineage":["https://openalex.org/I154130895"]}],"countries":["NZ"],"is_corresponding":false,"raw_author_name":"Yun Sing Koh","raw_affiliation_strings":["University of Auckland, Auckland, New Zealand"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Auckland, Auckland, New Zealand","institution_ids":["https://openalex.org/I154130895"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5041573811","display_name":"Vimala Balakrishnan","orcid":"https://orcid.org/0000-0002-6859-4488"},"institutions":[{"id":"https://openalex.org/I33849332","display_name":"University of Malaya","ror":"https://ror.org/00rzspn62","country_code":"MY","type":"education","lineage":["https://openalex.org/I33849332"]}],"countries":["MY"],"is_corresponding":false,"raw_author_name":"Vimala Balakrishnan","raw_affiliation_strings":["University of Malaya, Kuala Lumpur, Malaysia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Malaya, Kuala Lumpur, Malaysia","institution_ids":["https://openalex.org/I33849332"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.9687,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.84471136,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":95},"biblio":{"volume":"71","issue":"1","first_page":"2","last_page":"17"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9764999747276306,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7995744943618774},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.7242699861526489},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7016669511795044},{"id":"https://openalex.org/keywords/readability","display_name":"Readability","score":0.6509311199188232},{"id":"https://openalex.org/keywords/outlier","display_name":"Outlier","score":0.5133101344108582},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.47870755195617676},{"id":"https://openalex.org/keywords/grading","display_name":"Grading (engineering)","score":0.4322478175163269},{"id":"https://openalex.org/keywords/originality","display_name":"Originality","score":0.42863017320632935},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3714224696159363},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.21165001392364502},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.12249639630317688}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7995744943618774},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.7242699861526489},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7016669511795044},{"id":"https://openalex.org/C2778143727","wikidata":"https://www.wikidata.org/wiki/Q1820650","display_name":"Readability","level":2,"score":0.6509311199188232},{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.5133101344108582},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.47870755195617676},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.4322478175163269},{"id":"https://openalex.org/C2776950860","wikidata":"https://www.wikidata.org/wiki/Q2914681","display_name":"Originality","level":3,"score":0.42863017320632935},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3714224696159363},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21165001392364502},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.12249639630317688},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C11012388","wikidata":"https://www.wikidata.org/wiki/Q170658","display_name":"Creativity","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C147176958","wikidata":"https://www.wikidata.org/wiki/Q77590","display_name":"Civil engineering","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1108/ajim-04-2018-0086","is_oa":false,"landing_page_url":"https://doi.org/10.1108/ajim-04-2018-0086","pdf_url":null,"source":{"id":"https://openalex.org/S4210189668","display_name":"Aslib Journal of Information Management","issn_l":"0001-253X","issn":["0001-253X","1758-3748","2050-3806","2050-3814"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319811","host_organization_name":"Emerald Publishing Limited","host_organization_lineage":["https://openalex.org/P4310319811"],"host_organization_lineage_names":["Emerald Publishing Limited"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Aslib Journal of Information Management","raw_type":"journal-article"},{"id":"pmh:oai:eprints.um.edu.my:20008","is_oa":false,"landing_page_url":"http://eprints.um.edu.my/20008/","pdf_url":null,"source":{"id":"https://openalex.org/S4306400330","display_name":"The University of Malaya Research Repository (University of Malaya)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I33849332","host_organization_name":"University of Malaya","host_organization_lineage":["https://openalex.org/I33849332"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W133394232","https://openalex.org/W1967370444","https://openalex.org/W1971609128","https://openalex.org/W1974548087","https://openalex.org/W1976076261","https://openalex.org/W1983164881","https://openalex.org/W1998381498","https://openalex.org/W2006667248","https://openalex.org/W2022995284","https://openalex.org/W2024230945","https://openalex.org/W2053584122","https://openalex.org/W2062236692","https://openalex.org/W2090715008","https://openalex.org/W2094607109","https://openalex.org/W2098119513","https://openalex.org/W2105105276","https://openalex.org/W2119461458","https://openalex.org/W2120308175","https://openalex.org/W2156186232","https://openalex.org/W2283315146","https://openalex.org/W2565007888","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W2041415459","https://openalex.org/W1964661231","https://openalex.org/W4254960163","https://openalex.org/W2593155302","https://openalex.org/W3110264473","https://openalex.org/W2072812638","https://openalex.org/W2370831213","https://openalex.org/W3126095231","https://openalex.org/W4285478380","https://openalex.org/W4213170381"],"abstract_inverted_index":{"Purpose":[0],"The":[1,22,87,116],"effort":[2,38,55,99,122,151,174],"in":[3,32,49,58,96,143],"addition":[4],"to":[5,18,28,53,135,149,165],"relevance":[6,34,56,69,123,175],"is":[7,27,90,139,147,163],"a":[8,30,74,91,126],"major":[9],"factor":[10],"for":[11,78],"satisfaction":[12],"and":[13,84,153],"utility":[14],"of":[15,24,65,109,113,132,157],"the":[16,19,44,47,107,110,144,155],"document":[17,80],"actual":[20],"user.":[21],"purpose":[23],"this":[25],"paper":[26],"propose":[29],"method":[31],"generating":[33],"judgments":[35,70,124],"that":[36,141],"incorporate":[37],"without":[39],"human":[40],"judges\u2019":[41],"involvement.":[42],"Then":[43],"study":[45],"determines":[46],"variation":[48],"system":[50,145],"rankings":[51,146],"due":[52,148],"low":[54,121,150,173],"judgment":[57],"evaluating":[59],"retrieval":[60,117,167],"systems":[61,118,168],"at":[62,169],"different":[63],"depth":[64,131,171],"evaluation.":[66],"Design/methodology/approach":[67],"Effort-based":[68],"are":[71],"generated":[72],"using":[73,120,172],"proposed":[75],"boxplot":[76,88],"approach":[77,89,95],"simple":[79,92],"features,":[81],"HTML":[82],"features":[83],"readability":[85],"features.":[86],"yet":[93],"repeatable":[94],"classifying":[97],"documents\u2019":[98],"while":[100],"ensuring":[101],"outlier":[102],"scores":[103],"do":[104],"not":[105,154],"skew":[106],"grading":[108],"entire":[111],"set":[112],"documents.":[114,159],"Findings":[115],"evaluation":[119,133],"has":[125],"stronger":[127],"influence":[128],"on":[129],"shallow":[130,170],"compared":[134],"deeper":[136],"depth.":[137],"It":[138],"proved":[140],"difference":[142],"documents":[152],"number":[156],"relevant":[158],"Originality/value":[160],"Hence,":[161],"it":[162],"crucial":[164],"evaluate":[166],"judgments.":[176]},"counts_by_year":[{"year":2021,"cited_by_count":2}],"updated_date":"2026-07-30T17:31:21.811387","created_date":"2025-10-10T00:00:00"}