{"id":"https://openalex.org/W1966155475","doi":"https://doi.org/10.1145/2661829.2661897","title":"A Generative Model for Generating Relevance Labels from Human Judgments and Click-Logs","display_name":"A Generative Model for Generating Relevance Labels from Human Judgments and Click-Logs","publication_year":2014,"publication_date":"2014-11-03","ids":{"openalex":"https://openalex.org/W1966155475","doi":"https://doi.org/10.1145/2661829.2661897","mag":"1966155475"},"language":"en","primary_location":{"id":"doi:10.1145/2661829.2661897","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2661829.2661897","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013062475","display_name":"Xugang Ye","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xugang Ye","raw_affiliation_strings":["Microsoft, Bellevue, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Bellevue, WA, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210108985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100338386","display_name":"Jingjing Li","orcid":"https://orcid.org/0000-0002-5504-2529"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jingjing Li","raw_affiliation_strings":["Microsoft, Bellevue, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Bellevue, WA, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210108985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078564598","display_name":"Zijie Qi","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zijie Qi","raw_affiliation_strings":["Microsoft, Bellevue, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Bellevue, WA, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210108985"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088039997","display_name":"Bingyue Peng","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bingyue Peng","raw_affiliation_strings":["Microsoft, Bellevue, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Bellevue, WA, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210108985"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048057712","display_name":"Dan Massey","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210108985","display_name":"Bellevue Hospital Center","ror":"https://ror.org/01ky34z31","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1283621791","https://openalex.org/I4210086933","https://openalex.org/I4210108985"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dan Massey","raw_affiliation_strings":["Microsoft, Bellevue, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Bellevue, WA, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210108985"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.8204,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.81180341,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"1907","last_page":"1910"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9904000163078308,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7813713550567627},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7719272375106812},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.5834641456604004},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.5654653310775757},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5604965090751648},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.5215566158294678},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5208240747451782},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.48392254114151},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.46282440423965454},{"id":"https://openalex.org/keywords/multinomial-distribution","display_name":"Multinomial distribution","score":0.4488329589366913},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.4348360300064087},{"id":"https://openalex.org/keywords/population","display_name":"Population","score":0.4241659939289093},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.36661654710769653},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.13565394282341003},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.13160410523414612}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7813713550567627},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7719272375106812},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.5834641456604004},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.5654653310775757},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5604965090751648},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.5215566158294678},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5208240747451782},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.48392254114151},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46282440423965454},{"id":"https://openalex.org/C192065140","wikidata":"https://www.wikidata.org/wiki/Q1147928","display_name":"Multinomial distribution","level":2,"score":0.4488329589366913},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.4348360300064087},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.4241659939289093},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.36661654710769653},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.13565394282341003},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.13160410523414612},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2661829.2661897","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2661829.2661897","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.5899999737739563,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W1992549066","https://openalex.org/W2026784708","https://openalex.org/W2069870183","https://openalex.org/W2090883204","https://openalex.org/W2099213975","https://openalex.org/W2106630408","https://openalex.org/W2109722477","https://openalex.org/W2115584760","https://openalex.org/W2145617001","https://openalex.org/W2150731624","https://openalex.org/W2151592910","https://openalex.org/W2154739689","https://openalex.org/W6676367512","https://openalex.org/W6677385034"],"related_works":["https://openalex.org/W4365211920","https://openalex.org/W3014948380","https://openalex.org/W4380551139","https://openalex.org/W4317695495","https://openalex.org/W4287117424","https://openalex.org/W4387506531","https://openalex.org/W4238433571","https://openalex.org/W2967848559","https://openalex.org/W4299831724","https://openalex.org/W4283803360"],"abstract_inverted_index":{"Lack":[0],"of":[1,14,39,64,108,217],"high":[2,24],"quality":[3],"relevance":[4,111,138,160,187],"labels":[5,45,112,188,208],"is":[6,32,58,67,118],"a":[7,36,61,96,105,121,136,149,171],"common":[8],"challenge":[9],"in":[10],"the":[11,23,29,43,52,54,73,110,145,152,158,162,166,181,184,191,205,210,214,218],"early":[12],"stage":[13],"search":[15],"engine":[16],"development.":[17],"In":[18,100],"media":[19,114],"search,":[20],"due":[21],"to":[22,87,94,203],"recruiting":[25],"and":[26,49,69,82,129,161,197,213],"training":[27,196],"cost,":[28],"labeling":[30],"process":[31],"usually":[33],"conducted":[34],"by":[35,169],"small":[37],"number":[38],"human":[40,126,207],"judges.":[41],"Consequently,":[42],"generated":[44,134],"are":[46],"often":[47],"limited":[48,206],"biased.":[50,71],"On":[51],"contrary,":[53],"click":[55,74,130,153],"data":[56,75,192],"that":[57,124,151],"extracted":[59],"from":[60,135],"large":[62],"population":[63],"real":[65],"users":[66],"massive":[68],"less":[70],"However,":[72],"also":[76],"contains":[77],"considerable":[78],"noise.":[79],"Therefore,":[80],"more":[81,83],"researchers":[84],"have":[85,198],"begun":[86],"focus":[88],"on":[89,120,156,180],"combining":[90],"those":[91],"two":[92],"resources":[93],"generate":[95],"better":[97],"ground-truth":[98],"approximation.":[99],"this":[101],"paper,":[102],"we":[103],"present":[104],"novel":[106],"method":[107,117],"generating":[109],"for":[113,194],"search.":[115],"The":[116,142],"based":[119],"generative":[122],"model":[123,143,167],"considers":[125,144],"judgment,":[127],"position,":[128],"status":[131,154],"as":[132],"observations":[133],"hidden":[137,159],"with":[139,148,175],"multinomial":[140],"prior.":[141],"position":[146],"bias":[147],"requirement":[150],"depends":[155],"both":[157],"position.":[163],"We":[164],"infer":[165],"parameters":[168],"using":[170,204],"Gibbs":[172],"sampling":[173],"procedure":[174],"hyper-parameter":[176],"optimization.":[177],"From":[178],"experiments":[179],"Xbox's":[182],"data,":[183],"newly":[185],"inferred":[186],"significantly":[189],"increase":[190],"volume":[193],"ranker":[195],"demonstrated":[199],"superior":[200],"performance":[201],"compared":[202],"only,":[209,212],"click-through-rates":[211],"heuristic":[215],"combination":[216],"two.":[219]},"counts_by_year":[{"year":2018,"cited_by_count":1},{"year":2016,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}