{"id":"https://openalex.org/W2100767640","doi":"https://doi.org/10.1145/2661829.2661913","title":"Machine-Assisted Search Preference Evaluation","display_name":"Machine-Assisted Search Preference Evaluation","publication_year":2014,"publication_date":"2014-11-03","ids":{"openalex":"https://openalex.org/W2100767640","doi":"https://doi.org/10.1145/2661829.2661913","mag":"2100767640"},"language":"en","primary_location":{"id":"doi:10.1145/2661829.2661913","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2661829.2661913","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5021000040","display_name":"Ahmed Hassan Awadallah","orcid":"https://orcid.org/0000-0001-6426-3537"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ahmed Hassan Awadallah","raw_affiliation_strings":["Microsoft, Redmond, WA, USA","Microsoft Redmond, WA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]},{"raw_affiliation_string":"Microsoft Redmond, WA, USA#TAB#","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108365460","display_name":"Imed Zitouni","orcid":null},"institutions":[{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]},{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Imed Zitouni","raw_affiliation_strings":["Microsoft, Bellevue, WA, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft, Bellevue, WA, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210108985"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5021000040"],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":1.5778,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.8840996,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"51","last_page":"60"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.8008309602737427},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7954727411270142},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.6977301836013794},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6560163497924805},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5737166404724121},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5225377082824707},{"id":"https://openalex.org/keywords/preference-learning","display_name":"Preference learning","score":0.5062369704246521},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.43733203411102295},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.42231130599975586},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4186818301677704},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.07141360640525818},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07056617736816406}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.8008309602737427},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7954727411270142},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.6977301836013794},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6560163497924805},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5737166404724121},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5225377082824707},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.5062369704246521},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43733203411102295},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42231130599975586},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4186818301677704},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.07141360640525818},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07056617736816406},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2661829.2661913","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2661829.2661913","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":49,"referenced_works":["https://openalex.org/W94637519","https://openalex.org/W136383123","https://openalex.org/W216097746","https://openalex.org/W959078620","https://openalex.org/W971432383","https://openalex.org/W971511625","https://openalex.org/W1018128817","https://openalex.org/W1484594537","https://openalex.org/W1490008437","https://openalex.org/W1509228788","https://openalex.org/W1966835268","https://openalex.org/W1980036597","https://openalex.org/W2006811637","https://openalex.org/W2024046085","https://openalex.org/W2027198686","https://openalex.org/W2027382829","https://openalex.org/W2048987619","https://openalex.org/W2058473041","https://openalex.org/W2069870183","https://openalex.org/W2094790959","https://openalex.org/W2097443371","https://openalex.org/W2104677329","https://openalex.org/W2116008435","https://openalex.org/W2117424365","https://openalex.org/W2119634292","https://openalex.org/W2123937625","https://openalex.org/W2125682643","https://openalex.org/W2125771191","https://openalex.org/W2132314908","https://openalex.org/W2133968343","https://openalex.org/W2152314154","https://openalex.org/W2156037541","https://openalex.org/W2156160882","https://openalex.org/W2156186232","https://openalex.org/W2158551114","https://openalex.org/W2159048649","https://openalex.org/W2160471965","https://openalex.org/W2165293013","https://openalex.org/W2166426462","https://openalex.org/W2170686633","https://openalex.org/W2171806843","https://openalex.org/W2278434201","https://openalex.org/W2283838425","https://openalex.org/W2399942877","https://openalex.org/W2757426349","https://openalex.org/W4285719527","https://openalex.org/W6603897584","https://openalex.org/W6677618753","https://openalex.org/W6744702772"],"related_works":["https://openalex.org/W2937325523","https://openalex.org/W2954428433","https://openalex.org/W4205377104","https://openalex.org/W149611507","https://openalex.org/W257970033","https://openalex.org/W1994181006","https://openalex.org/W2911102221","https://openalex.org/W4285602503","https://openalex.org/W4281387587","https://openalex.org/W2943672508"],"abstract_inverted_index":{"Information":[0],"Retrieval":[1],"systems":[2],"are":[3],"traditionally":[4],"evaluated":[5],"using":[6],"the":[7,24,46,53,76,128,134],"relevance":[8],"of":[9,26,48,130],"web":[10],"pages":[11],"to":[12,66,72,103,154],"individual":[13],"queries.":[14],"Other":[15],"work":[16],"on":[17,22,133,139],"IR":[18],"evaluation":[19],"has":[20],"focused":[21],"exploring":[23],"use":[25],"preference":[27,39,61,81,108,124],"judgments":[28,40,62,82,109],"over":[29,41],"two":[30,42],"search":[31,43,56],"result":[32,79,106],"lists.":[33],"Unlike":[34],"traditional":[35],"query-document":[36],"evaluation,":[37],"collecting":[38],"result-lists":[44],"takes":[45,52],"context":[47],"documents,":[49],"and":[50,145],"hence":[51],"interaction":[54],"between":[55],"results,":[57],"into":[58],"consideration.":[59],"Moreover,":[60],"have":[63,83],"been":[64],"shown":[65],"produce":[67],"more":[68],"accurate":[69],"results":[70],"compared":[71],"absolute":[73],"judgment.":[74],"On":[75],"other":[77],"hand":[78],"list":[80,107],"very":[84],"high":[85,143],"annotation":[86,163],"cost.":[87,164],"In":[88],"this":[89],"work,":[90],"we":[91],"investigate":[92,127],"how":[93],"machine":[94],"learned":[95],"models":[96,119,149],"can":[97,121,150],"assist":[98,155],"human":[99,156],"judges":[100,157],"in":[101,159,162],"order":[102],"collect":[104],"reliable":[105],"at":[110],"large":[111],"scale":[112],"with":[113,142],"lower":[114],"judgment-cost.":[115],"We":[116,126,137],"build":[117],"novel":[118],"that":[120,147],"predict":[122],"user":[123],"automatically.":[125],"effect":[129],"different":[131],"features":[132],"prediction":[135],"quality.":[136],"focus":[138],"predicting":[140],"preferences":[141],"confidence":[144],"show":[146],"these":[148],"be":[151],"effectively":[152],"used":[153],"resulting":[158],"significant":[160],"reduction":[161]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2017,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}