{"id":"https://openalex.org/W3156817235","doi":"https://doi.org/10.1145/3404835.3462784","title":"DiffIR: Exploring Differences in Ranking Models' Behavior","display_name":"DiffIR: Exploring Differences in Ranking Models' Behavior","publication_year":2021,"publication_date":"2021-07-11","ids":{"openalex":"https://openalex.org/W3156817235","doi":"https://doi.org/10.1145/3404835.3462784","mag":"3156817235"},"language":"en","primary_location":{"id":"doi:10.1145/3404835.3462784","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3404835.3462784","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3404835.3462784","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3404835.3462784","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5059380000","display_name":"Kevin Martin Jose","orcid":null},"institutions":[{"id":"https://openalex.org/I4210109712","display_name":"Max Planck Institute for Informatics","ror":"https://ror.org/01w19ak89","country_code":"DE","type":"facility","lineage":["https://openalex.org/I149899117","https://openalex.org/I4210109712"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Kevin Martin Jose","raw_affiliation_strings":["Max Planck Institute for Informatics, Saarbruecken, Germany"],"affiliations":[{"raw_affiliation_string":"Max Planck Institute for Informatics, Saarbruecken, Germany","institution_ids":["https://openalex.org/I4210109712"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101561868","display_name":"T. Q. Nguyen","orcid":"https://orcid.org/0000-0003-3954-5131"},"institutions":[{"id":"https://openalex.org/I4210109712","display_name":"Max Planck Institute for Informatics","ror":"https://ror.org/01w19ak89","country_code":"DE","type":"facility","lineage":["https://openalex.org/I149899117","https://openalex.org/I4210109712"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Thong Nguyen","raw_affiliation_strings":["Max Planck Institute for Informatics, Saarbruecken, Germany"],"affiliations":[{"raw_affiliation_string":"Max Planck Institute for Informatics, Saarbruecken, Germany","institution_ids":["https://openalex.org/I4210109712"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014199889","display_name":"Sean MacAvaney","orcid":"https://orcid.org/0000-0002-8914-2659"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Sean MacAvaney","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071842569","display_name":"Jeff Dalton","orcid":"https://orcid.org/0000-0003-2422-8651"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jeffrey Dalton","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5059489981","display_name":"Andrew Yates","orcid":"https://orcid.org/0000-0002-5970-880X"},"institutions":[{"id":"https://openalex.org/I4210109712","display_name":"Max Planck Institute for Informatics","ror":"https://ror.org/01w19ak89","country_code":"DE","type":"facility","lineage":["https://openalex.org/I149899117","https://openalex.org/I4210109712"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Andrew Yates","raw_affiliation_strings":["Max Planck Institute for Informatics, Saarbruecken, Germany"],"affiliations":[{"raw_affiliation_string":"Max Planck Institute for Informatics, Saarbruecken, Germany","institution_ids":["https://openalex.org/I4210109712"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5059380000"],"corresponding_institution_ids":["https://openalex.org/I4210109712"],"apc_list":null,"apc_paid":null,"fwci":1.118,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.817987,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"2595","last_page":"2599"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9883000254631042,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.818017840385437},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7909864783287048},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6528602838516235},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.6457589268684387},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.6161800622940063},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5885046720504761},{"id":"https://openalex.org/keywords/complement","display_name":"Complement (music)","score":0.5331850051879883},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.509145200252533},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.48243770003318787},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4769919514656067},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.47250163555145264},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.1339903473854065}],"concepts":[{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.818017840385437},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7909864783287048},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6528602838516235},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.6457589268684387},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.6161800622940063},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5885046720504761},{"id":"https://openalex.org/C112313634","wikidata":"https://www.wikidata.org/wiki/Q7886648","display_name":"Complement (music)","level":5,"score":0.5331850051879883},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.509145200252533},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.48243770003318787},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4769919514656067},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.47250163555145264},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.1339903473854065},{"id":"https://openalex.org/C188082640","wikidata":"https://www.wikidata.org/wiki/Q1780899","display_name":"Complementation","level":4,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C127716648","wikidata":"https://www.wikidata.org/wiki/Q104053","display_name":"Phenotype","level":3,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3404835.3462784","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3404835.3462784","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3404835.3462784","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:dare.uva.nl:openaire_cris_publications/b2424e05-e8e6-4430-ae44-8cdada91b511","is_oa":false,"landing_page_url":"https://hdl.handle.net/11245.1/b2424e05-e8e6-4430-ae44-8cdada91b511","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Jose, K M, Nguyen, T, MacAvaney, S, Dalton, J & Yates, A 2021, DiffIR: Exploring Differences in Ranking Models' Behavior. in SIGIR '21 : proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval : July 11-15, 2021, virtual event, Canada. Association for Computing Machinery, New York, NY, pp. 2595-2599, 44th International ACM SIGIR Conference on Research and Development in Information Retrieval, Montr\u00e9al, Quebec, Canada, 11/07/21. https://doi.org/10.1145/3404835.3462784","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:dare.uva.nl:publications/b2424e05-e8e6-4430-ae44-8cdada91b511","is_oa":false,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/diffir-exploring-differences-in-ranking-models-behavior(b2424e05-e8e6-4430-ae44-8cdada91b511).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Jose, K M, Nguyen, T, MacAvaney, S, Dalton, J & Yates, A 2021, DiffIR: Exploring Differences in Ranking Models' Behavior. in SIGIR '21 : proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval : July 11-15, 2021, virtual event, Canada. Association for Computing Machinery, New York, NY, pp. 2595-2599, 44th International ACM SIGIR Conference on Research and Development in Information Retrieval, Montr\u00e9al, Quebec, Canada, 11/07/21. https://doi.org/10.1145/3404835.3462784","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:eprints.gla.ac.uk:239078","is_oa":true,"landing_page_url":"http://eprints.gla.ac.uk/239078/","pdf_url":null,"source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"}],"best_oa_location":{"id":"doi:10.1145/3404835.3462784","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3404835.3462784","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3404835.3462784","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.44999998807907104}],"awards":[{"id":"https://openalex.org/G7302224536","display_name":null,"funder_award_id":"EP/V025708/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3156817235.pdf","grobid_xml":"https://content.openalex.org/works/W3156817235.grobid-xml"},"referenced_works_count":42,"referenced_works":["https://openalex.org/W98855157","https://openalex.org/W118024169","https://openalex.org/W193079019","https://openalex.org/W1707848225","https://openalex.org/W1966835268","https://openalex.org/W1976076261","https://openalex.org/W2004918107","https://openalex.org/W2006811637","https://openalex.org/W2017833987","https://openalex.org/W2021581601","https://openalex.org/W2082359527","https://openalex.org/W2084567515","https://openalex.org/W2099964107","https://openalex.org/W2116008435","https://openalex.org/W2124504084","https://openalex.org/W2133968343","https://openalex.org/W2166426462","https://openalex.org/W2346933546","https://openalex.org/W2469178887","https://openalex.org/W2536015822","https://openalex.org/W2648699835","https://openalex.org/W2782157559","https://openalex.org/W2794432940","https://openalex.org/W2798482016","https://openalex.org/W2897412906","https://openalex.org/W2927996276","https://openalex.org/W2940927814","https://openalex.org/W2945127593","https://openalex.org/W2968287414","https://openalex.org/W2971209824","https://openalex.org/W2982096936","https://openalex.org/W3001344098","https://openalex.org/W3021779606","https://openalex.org/W3045745713","https://openalex.org/W3094297635","https://openalex.org/W3099384026","https://openalex.org/W3099446234","https://openalex.org/W3102286003","https://openalex.org/W3103251620","https://openalex.org/W3104914963","https://openalex.org/W3133594109","https://openalex.org/W3156166379"],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W2745001401","https://openalex.org/W4321353415","https://openalex.org/W2130974462","https://openalex.org/W972276598","https://openalex.org/W4246352526","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W2087343574","https://openalex.org/W2121910908"],"abstract_inverted_index":{"Understanding":[0],"and":[1,19,89],"comparing":[2],"the":[3,56,78,97,117,168],"behavior":[4,33,63,118],"of":[5,68,80,119,164],"retrieval":[6,125],"models":[7,81,176],"is":[8,38],"a":[9,39,91,103,133,139,178],"fundamental":[10],"challenge":[11],"that":[12,127],"requires":[13],"going":[14],"beyond":[15],"examining":[16],"average":[17],"effectiveness":[18],"per-query":[20],"metrics,":[21],"because":[22],"these":[23],"do":[24],"not":[25],"reveal":[26],"key":[27],"differences":[28,85,170],"in":[29,86],"how":[30],"ranking":[31,48,175],"models'":[32],"impacts":[34],"individual":[35,57,87],"results.":[36],"DiffIR":[37,100,165],"new":[40],"open-source":[41],"web":[42,93],"tool":[43,146],"to":[44,95,154],"assist":[45],"with":[46],"qualitative":[47,169],"analysis":[49],"by":[50,166],"visually":[51],"'diffing'":[52],"system":[53],"rankings":[54,79,88,98],"at":[55],"result":[58],"level":[59],"for":[60,76],"queries":[61,75],"where":[62],"significantly":[64],"diverges.":[65],"Using":[66],"one":[67],"several":[69],"configurable":[70],"similarity":[71,134],"measures,":[72],"it":[73],"identifies":[74],"which":[77],"compared":[82],"have":[83],"important":[84],"provides":[90],"visual":[92],"interface":[94],"compare":[96],"side-by-side.":[99],"additionally":[101],"supports":[102],"model-specific":[104],"visualization":[105],"approach":[106],"based":[107,131,137],"on":[108,132,138,177],"custom":[109],"term":[110],"importance":[111],"weight":[112],"files.":[113],"These":[114],"support":[115],"studying":[116,167],"interpretable":[120],"models,":[121],"such":[122],"as":[123],"neural":[124,149,174],"methods":[126],"produce":[128],"document":[129,141],"scores":[130],"matrix":[135],"or":[136],"single":[140],"passage.":[142],"Observations":[143],"from":[144],"this":[145],"can":[147],"complement":[148],"probing":[150],"approaches":[151],"like":[152],"ABNIRML":[153],"generate":[155],"quantitative":[156],"tests.":[157],"We":[158],"provide":[159],"an":[160],"illustrative":[161],"use":[162],"case":[163],"between":[171],"recently":[172],"developed":[173],"standard":[179],"TREC":[180],"benchmark":[181],"dataset.":[182]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-03-17T09:09:15.849793","created_date":"2025-10-10T00:00:00"}