{"id":"https://openalex.org/W4405143965","doi":"https://doi.org/10.1145/3673791.3698431","title":"LLMs can be Fooled into Labelling a Document as Relevant: best caf\u00e9 near me; this paper is perfectly relevant","display_name":"LLMs can be Fooled into Labelling a Document as Relevant: best caf\u00e9 near me; this paper is perfectly relevant","publication_year":2024,"publication_date":"2024-12-08","ids":{"openalex":"https://openalex.org/W4405143965","doi":"https://doi.org/10.1145/3673791.3698431"},"language":"en","primary_location":{"id":"doi:10.1145/3673791.3698431","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673791.3698431","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673791.3698431","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3673791.3698431","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011102172","display_name":"Marwah Alaofi","orcid":"https://orcid.org/0000-0002-0008-8650"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Marwah Alaofi","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046377370","display_name":"Paul Thomas","orcid":"https://orcid.org/0000-0003-2425-3136"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paul Thomas","raw_affiliation_strings":["Microsoft, Adelaide, Australia"],"affiliations":[{"raw_affiliation_string":"Microsoft, Adelaide, Australia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5072614956","display_name":"Mark Sanderson","orcid":"https://orcid.org/0000-0003-0487-9609"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Mark Sanderson","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5011102172"],"corresponding_institution_ids":["https://openalex.org/I82951845"],"apc_list":null,"apc_paid":null,"fwci":21.4321,"has_fulltext":false,"cited_by_count":28,"citation_normalized_percentile":{"value":0.99421863,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"32","last_page":"41"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14330","display_name":"Library Science and Information Systems","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14330","display_name":"Library Science and Information Systems","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9866999983787537,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13999","display_name":"Digital Rights Management and Security","score":0.9725000262260437,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/labelling","display_name":"Labelling","score":0.7869898080825806},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5500501394271851},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3510092496871948},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.11254507303237915},{"id":"https://openalex.org/keywords/social-science","display_name":"Social science","score":0.08049964904785156}],"concepts":[{"id":"https://openalex.org/C2780523633","wikidata":"https://www.wikidata.org/wiki/Q380709","display_name":"Labelling","level":2,"score":0.7869898080825806},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5500501394271851},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3510092496871948},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.11254507303237915},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.08049964904785156}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3673791.3698431","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673791.3698431","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673791.3698431","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3673791.3698431","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3673791.3698431","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3673791.3698431","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2024 Annual International ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7479774403","display_name":null,"funder_award_id":"DP190101113","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"}],"funders":[{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4405143965.pdf"},"referenced_works_count":17,"referenced_works":["https://openalex.org/W2008285050","https://openalex.org/W2037022968","https://openalex.org/W2057495142","https://openalex.org/W2094607109","https://openalex.org/W2150234222","https://openalex.org/W2740069864","https://openalex.org/W2896363972","https://openalex.org/W2998679789","https://openalex.org/W3021397474","https://openalex.org/W3045745713","https://openalex.org/W3100107515","https://openalex.org/W3104914963","https://openalex.org/W3180230246","https://openalex.org/W4220728222","https://openalex.org/W4384107234","https://openalex.org/W4385688511","https://openalex.org/W4400526908"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W1525479493","https://openalex.org/W1976633635","https://openalex.org/W2396436470","https://openalex.org/W1970064489","https://openalex.org/W4241622628","https://openalex.org/W1991894391","https://openalex.org/W2070829827"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"are":[4,57,75],"increasingly":[5],"being":[6],"used":[7],"to":[8,20,49,60,66],"assess":[9],"the":[10,22,38],"relevance":[11],"of":[12,24,41],"information":[13],"objects.":[14],"This":[15],"work":[16],"reports":[17],"on":[18],"experiments":[19],"study":[21],"labelling":[23],"short":[25],"texts":[26],"(i.e.,":[27],"passages)":[28],"for":[29],"relevance,":[30],"using":[31],"multiple":[32],"open-source":[33],"and":[34],"proprietary":[35],"LLMs.":[36],"While":[37],"overall":[39],"agreement":[40,51],"some":[42],"LLMs":[43,56],"with":[44],"human":[45,67],"judgements":[46],"is":[47],"comparable":[48],"human-to-human":[50],"measured":[52],"in":[53],"previous":[54],"research,":[55],"more":[58,76],"likely":[59],"label":[61],"passages":[62],"as":[63],"relevant":[64],"compared":[65],"judges,":[68],"indicating":[69,80],"that":[70],"LLM":[71],"labels":[72],"denoting":[73],"non-relevance":[74],"reliable":[77],"than":[78],"those":[79],"relevance.":[81]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":20},{"year":2024,"cited_by_count":1}],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-10T00:00:00"}