{"id":"https://openalex.org/W4400977018","doi":"https://doi.org/10.1145/3681786","title":"Online and Offline Evaluation in Search Clarification","display_name":"Online and Offline Evaluation in Search Clarification","publication_year":2024,"publication_date":"2024-07-25","ids":{"openalex":"https://openalex.org/W4400977018","doi":"https://doi.org/10.1145/3681786"},"language":"en","primary_location":{"id":"doi:10.1145/3681786","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3681786","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3681786","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3681786","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5017632892","display_name":"Leila Tavakoli","orcid":"https://orcid.org/0000-0002-5951-4052"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Leila Tavakoli","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":"https://orcid.org/0000-0002-5951-4052","affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053547177","display_name":"Johanne R. Trippas","orcid":"https://orcid.org/0000-0002-7801-0239"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Johanne R. Trippas","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":"https://orcid.org/0000-0002-7801-0239","affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101457713","display_name":"Hamed Zamani","orcid":"https://orcid.org/0000-0002-0800-3340"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hamed Zamani","raw_affiliation_strings":["University of Massachusetts Amherst, Amherst, MA, USA"],"raw_orcid":"https://orcid.org/0000-0002-0800-3340","affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, Amherst, MA, USA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085925326","display_name":"Falk Scholer","orcid":"https://orcid.org/0000-0001-9094-0810"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Falk Scholer","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":"https://orcid.org/0000-0001-9094-0810","affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5072614956","display_name":"Mark Sanderson","orcid":"https://orcid.org/0000-0003-0487-9609"},"institutions":[{"id":"https://openalex.org/I82951845","display_name":"RMIT University","ror":"https://ror.org/04ttjf776","country_code":"AU","type":"education","lineage":["https://openalex.org/I82951845"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Mark Sanderson","raw_affiliation_strings":["RMIT University, Melbourne, Australia"],"raw_orcid":"https://orcid.org/0000-0003-0487-9609","affiliations":[{"raw_affiliation_string":"RMIT University, Melbourne, Australia","institution_ids":["https://openalex.org/I82951845"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.16474618,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"43","issue":"1","first_page":"1","last_page":"30"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7308081388473511},{"id":"https://openalex.org/keywords/online-and-offline","display_name":"Online and offline","score":0.7084647417068481},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.6782705783843994},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.6687769889831543},{"id":"https://openalex.org/keywords/offline-learning","display_name":"Offline learning","score":0.4931604266166687},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.48160693049430847},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4696573317050934},{"id":"https://openalex.org/keywords/user-engagement","display_name":"User engagement","score":0.46765369176864624},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.36742544174194336},{"id":"https://openalex.org/keywords/online-learning","display_name":"Online learning","score":0.3510483503341675},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.33631402254104614},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.27474191784858704}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7308081388473511},{"id":"https://openalex.org/C2780102126","wikidata":"https://www.wikidata.org/wiki/Q10928179","display_name":"Online and offline","level":2,"score":0.7084647417068481},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.6782705783843994},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.6687769889831543},{"id":"https://openalex.org/C2780490138","wikidata":"https://www.wikidata.org/wiki/Q7079636","display_name":"Offline learning","level":3,"score":0.4931604266166687},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.48160693049430847},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4696573317050934},{"id":"https://openalex.org/C2984870255","wikidata":"https://www.wikidata.org/wiki/Q5196451","display_name":"User engagement","level":2,"score":0.46765369176864624},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36742544174194336},{"id":"https://openalex.org/C2986087404","wikidata":"https://www.wikidata.org/wiki/Q15946010","display_name":"Online learning","level":2,"score":0.3510483503341675},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33631402254104614},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.27474191784858704},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3681786","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3681786","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3681786","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3681786","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3681786","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3681786","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.4399999976158142}],"awards":[{"id":"https://openalex.org/G2759510716","display_name":"Finding answers for complex questions","funder_award_id":"DP180102687","funder_id":"https://openalex.org/F4320334704","funder_display_name":"Australian Research Council"}],"funders":[{"id":"https://openalex.org/F4320334704","display_name":"Australian Research Council","ror":"https://ror.org/05mmh0f86"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4400977018.pdf"},"referenced_works_count":73,"referenced_works":["https://openalex.org/W1495821370","https://openalex.org/W1968528267","https://openalex.org/W1968927634","https://openalex.org/W1972802558","https://openalex.org/W1974360117","https://openalex.org/W1978658266","https://openalex.org/W1982204215","https://openalex.org/W1982530130","https://openalex.org/W1998381498","https://openalex.org/W2000246295","https://openalex.org/W2005825070","https://openalex.org/W2010953147","https://openalex.org/W2017329135","https://openalex.org/W2018371603","https://openalex.org/W2021581601","https://openalex.org/W2031322820","https://openalex.org/W2031489019","https://openalex.org/W2035720976","https://openalex.org/W2037140704","https://openalex.org/W2040927952","https://openalex.org/W2047221353","https://openalex.org/W2061503185","https://openalex.org/W2078488685","https://openalex.org/W2099213975","https://openalex.org/W2110228583","https://openalex.org/W2117303881","https://openalex.org/W2123937625","https://openalex.org/W2134557242","https://openalex.org/W2138216384","https://openalex.org/W2153274408","https://openalex.org/W2158254843","https://openalex.org/W2158312787","https://openalex.org/W2163162311","https://openalex.org/W2164363339","https://openalex.org/W2207136466","https://openalex.org/W2318802957","https://openalex.org/W2340526403","https://openalex.org/W2511414107","https://openalex.org/W2511702311","https://openalex.org/W2593527188","https://openalex.org/W2604541358","https://openalex.org/W2740916867","https://openalex.org/W2798460079","https://openalex.org/W2798863366","https://openalex.org/W2892181857","https://openalex.org/W2917637360","https://openalex.org/W2955157382","https://openalex.org/W2963033005","https://openalex.org/W2971169991","https://openalex.org/W2997842202","https://openalex.org/W3003611599","https://openalex.org/W3008379384","https://openalex.org/W3034920311","https://openalex.org/W3047936157","https://openalex.org/W3074433075","https://openalex.org/W3094347401","https://openalex.org/W3096844885","https://openalex.org/W3100591766","https://openalex.org/W3100598124","https://openalex.org/W3104499181","https://openalex.org/W3147653952","https://openalex.org/W3150250864","https://openalex.org/W3195872342","https://openalex.org/W3198952405","https://openalex.org/W3208649236","https://openalex.org/W4246147481","https://openalex.org/W4281763369","https://openalex.org/W4287996196","https://openalex.org/W4290723744","https://openalex.org/W4310015411","https://openalex.org/W4366422585","https://openalex.org/W4385570600","https://openalex.org/W4388042901"],"related_works":["https://openalex.org/W4221145086","https://openalex.org/W4226221094","https://openalex.org/W4387293922","https://openalex.org/W4225619808","https://openalex.org/W4388482290","https://openalex.org/W4388872820","https://openalex.org/W4220668416","https://openalex.org/W4318719276","https://openalex.org/W4382463554","https://openalex.org/W3207447243"],"abstract_inverted_index":{"The":[0],"effectiveness":[1],"of":[2,25,47,103,125,193,209],"clarification":[3,48,104,137,181,195,223],"question":[4],"models":[5,217,241,262],"in":[6,67,79,170,218],"engaging":[7,136,180,222],"users":[8,41],"within":[9],"search":[10,80],"systems":[11],"is":[12,29],"currently":[13],"constrained,":[14],"casting":[15],"doubt":[16],"on":[17,111],"their":[18],"overall":[19],"usefulness.":[20],"To":[21],"improve":[22],"the":[23,45,57,99,106,117,134,140,154,160,194,207,220,226,256],"performance":[24,208],"these":[26],"models,":[27],"it":[28],"crucial":[30],"to":[31,73,94,96,116,183],"employ":[32,90],"assessment":[33,53],"approaches":[34],"that":[35,120,131,153,178,239],"encompass":[36],"both":[37],"real-time":[38],"feedback":[39],"from":[40,139,185,225],"(online":[42],"evaluation)":[43],"and":[44,61,89,144,164,167,188,191,214,263],"characteristics":[46,192],"questions":[49,138,224],"evaluated":[50],"through":[51],"human":[52,203],"(offline":[54],"evaluation).":[55],"However,":[56,250],"relationship":[58,161],"between":[59,162],"online":[60,112,127,143,163,171],"offline":[62,92,100,121,145,165,231,248,264],"evaluations":[63,122,146,232],"has":[64],"been":[65],"debated":[66],"information":[68],"retrieval.":[69],"This":[70],"study":[71],"aims":[72],"investigate":[74,95,201],"how":[75],"this":[76,174],"discordance":[77],"holds":[78],"clarification.":[81],"We":[82,151,176,199],"use":[83],"user":[84,113],"engagement":[85],"as":[86,233,255],"ground":[87],"truth":[88],"several":[91],"labels":[93,204],"what":[97],"extent":[98],"ranked":[101,108],"lists":[102,109],"resemble":[105],"ideal":[107],"based":[110],"engagement.":[114],"Contrary":[115],"current":[118],"understanding":[119],"fall":[123],"short":[124],"supporting":[126],"evaluations,":[128,166],"we":[129],"indicate":[130,238],"when":[132],"identifying":[133,219],"most":[135,221],"user\u2019s":[141,227],"perspective,":[142],"correspond":[147],"with":[148],"each":[149],"other.":[150],"show":[152],"query":[155],"length":[156],"does":[157],"not":[158,243],"influence":[159],"reducing":[168],"uncertainty":[169],"evaluation":[172],"strengthens":[173],"relationship.":[175],"illustrate":[177],"an":[179,252],"needs":[182],"excel":[184],"multiple":[186],"perspectives,":[187],"SERP":[189],"quality":[190],"are":[196],"equally":[197],"important.":[198],"also":[200],"if":[202],"can":[205],"enhance":[206],"Large":[210],"Language":[211],"Models":[212],"(LLMs)":[213],"Learning-to-Rank":[215],"(LTR)":[216],"perspective":[228],"by":[229],"incorporating":[230],"input":[234],"features.":[235],"Our":[236],"results":[237],"LTR":[240,261],"do":[242],"perform":[244],"better":[245],"than":[246],"individual":[247],"labels.":[249,265],"GPT,":[251],"LLM,":[253],"emerges":[254],"standout":[257],"performer,":[258],"surpassing":[259],"all":[260]},"counts_by_year":[],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}