{"id":"https://openalex.org/W3020834808","doi":"https://doi.org/10.1145/3397271.3401094","title":"Training Curricula for Open Domain Answer Re-Ranking","display_name":"Training Curricula for Open Domain Answer Re-Ranking","publication_year":2020,"publication_date":"2020-07-25","ids":{"openalex":"https://openalex.org/W3020834808","doi":"https://doi.org/10.1145/3397271.3401094","mag":"3020834808"},"language":"en","primary_location":{"id":"doi:10.1145/3397271.3401094","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3397271.3401094","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2004.14269","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Sean MacAvaney","orcid":null},"institutions":[{"id":"https://openalex.org/I184565670","display_name":"Georgetown University","ror":"https://ror.org/05vzafd60","country_code":"US","type":"education","lineage":["https://openalex.org/I184565670"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sean MacAvaney","raw_affiliation_strings":["Georgetown University, Washington, DC, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgetown University, Washington, DC, USA","institution_ids":["https://openalex.org/I184565670"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Franco Maria Nardini","orcid":null},"institutions":[{"id":"https://openalex.org/I122991210","display_name":"Istituto di Scienza e Tecnologie dell'Informazione \"Alessandro Faedo\"","ror":"https://ror.org/05kacka20","country_code":"IT","type":"facility","lineage":["https://openalex.org/I122991210","https://openalex.org/I4210155236"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Franco Maria Nardini","raw_affiliation_strings":["ISTI-CNR, Pisa, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ISTI-CNR, Pisa, Italy","institution_ids":["https://openalex.org/I122991210"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Raffaele Perego","orcid":null},"institutions":[{"id":"https://openalex.org/I122991210","display_name":"Istituto di Scienza e Tecnologie dell'Informazione \"Alessandro Faedo\"","ror":"https://ror.org/05kacka20","country_code":"IT","type":"facility","lineage":["https://openalex.org/I122991210","https://openalex.org/I4210155236"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Raffaele Perego","raw_affiliation_strings":["ISTI-CNR, Pisa, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ISTI-CNR, Pisa, Italy","institution_ids":["https://openalex.org/I122991210"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Nicola Tonellotto","orcid":null},"institutions":[{"id":"https://openalex.org/I108290504","display_name":"University of Pisa","ror":"https://ror.org/03ad39j10","country_code":"IT","type":"education","lineage":["https://openalex.org/I108290504"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Nicola Tonellotto","raw_affiliation_strings":["University of Pisa, Pisa, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Pisa, Pisa, Italy","institution_ids":["https://openalex.org/I108290504"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Nazli Goharian","orcid":null},"institutions":[{"id":"https://openalex.org/I184565670","display_name":"Georgetown University","ror":"https://ror.org/05vzafd60","country_code":"US","type":"education","lineage":["https://openalex.org/I184565670"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nazli Goharian","raw_affiliation_strings":["Georgetown University, Washington, DC, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgetown University, Washington, DC, USA","institution_ids":["https://openalex.org/I184565670"]}]},{"author_position":"last","author":{"id":null,"display_name":"Ophir Frieder","orcid":null},"institutions":[{"id":"https://openalex.org/I184565670","display_name":"Georgetown University","ror":"https://ror.org/05vzafd60","country_code":"US","type":"education","lineage":["https://openalex.org/I184565670"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ophir Frieder","raw_affiliation_strings":["Georgetown University, Washington, DC, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgetown University, Washington, DC, USA","institution_ids":["https://openalex.org/I184565670"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"529","last_page":"538"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6970999836921692},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.690500020980835},{"id":"https://openalex.org/keywords/pointwise","display_name":"Pointwise","score":0.6021999716758728},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.49810001254081726},{"id":"https://openalex.org/keywords/weighting","display_name":"Weighting","score":0.4916999936103821},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.4903999865055084},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.48739999532699585},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.48410001397132874}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.699400007724762},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6970999836921692},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.690500020980835},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.633899986743927},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6189000010490417},{"id":"https://openalex.org/C2777984123","wikidata":"https://www.wikidata.org/wiki/Q9248237","display_name":"Pointwise","level":2,"score":0.6021999716758728},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.49810001254081726},{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.4916999936103821},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.4903999865055084},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.48739999532699585},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.48410001397132874},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.4690000116825104},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4602999985218048},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.44859999418258667},{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.40790000557899475},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.39899998903274536},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.367900013923645},{"id":"https://openalex.org/C108010975","wikidata":"https://www.wikidata.org/wiki/Q500094","display_name":"Pruning","level":2,"score":0.3203999996185303},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.30250000953674316},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2978000044822693},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2581000030040741}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3397271.3401094","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3397271.3401094","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 43rd International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2004.14269","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2004.14269","pdf_url":"https://arxiv.org/pdf/2004.14269","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:arpi.unipi.it:11568/1054905","is_oa":false,"landing_page_url":"http://hdl.handle.net/11568/1054905","pdf_url":null,"source":{"id":"https://openalex.org/S4377196265","display_name":"CINECA IRIS Institutial research information system (University of Pisa)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I108290504","host_organization_name":"University of Pisa","host_organization_lineage":["https://openalex.org/I108290504"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:dnet:people______::d39b04efa4f5b304fc93a39f9c3b05c9","is_oa":true,"landing_page_url":"https://openportal.isti.cnr.it/doc?id=people______::d39b04efa4f5b304fc93a39f9c3b05c9","pdf_url":null,"source":{"id":"https://openalex.org/S7407055261","display_name":"ISTI Open Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"43rd International ACM SIGIR Conference on Research and Development in Information Retrieval, pp. 529\u2013538, online, 25-30 July, 2020","raw_type":"http://purl.org/coar/resource_type/c_5794"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2004.14269","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2004.14269","pdf_url":"https://arxiv.org/pdf/2004.14269","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W2057218950","https://openalex.org/W2070740689","https://openalex.org/W2186845332","https://openalex.org/W2296073425","https://openalex.org/W2493916176","https://openalex.org/W2766284073","https://openalex.org/W2782157559","https://openalex.org/W2782554945","https://openalex.org/W2783640434","https://openalex.org/W2897435371","https://openalex.org/W2899154813","https://openalex.org/W2910577570","https://openalex.org/W2940927814","https://openalex.org/W2994846609","https://openalex.org/W4288280763","https://openalex.org/W6683738474"],"related_works":[],"abstract_inverted_index":{"In":[0,82],"precision-oriented":[1],"tasks":[2],"like":[3],"answer":[4,94,163],"ranking,":[5],"it":[6],"is":[7],"more":[8,70,232],"important":[9],"to":[10,17,31,34,47,61,73,89,103,121,144,172,193,201,215,231],"rank":[11],"many":[12],"relevant":[13,20],"answers":[14,39,48],"highly":[15],"than":[16],"retrieve":[18],"all":[19,146],"answers.":[21],"It":[22],"follows":[23],"that":[24,49,52,65,114,126,168,226],"a":[25,43,57,108,123,154,194,202,208,220],"good":[26],"ranking":[27,45,59,164,179],"strategy":[28],"would":[29],"be":[30,119],"learn":[32],"how":[33],"identify":[35],"the":[36,90,105,115,132,136,216],"easiest":[37],"correct":[38],"first":[40],"(i.e.,":[41],"assign":[42],"high":[44],"score":[46,60],"have":[50],"characteristics":[51,64],"usually":[53],"indicate":[54],"relevance,":[55],"and":[56,183,188,207],"low":[58],"those":[62],"with":[63],"do":[66],"not),":[67],"before":[68],"incorporating":[69],"complex":[71],"logic":[72],"handle":[74],"difficult":[75,128],"cases":[76],"(e.g.,":[77],"semantic":[78],"matching":[79],"or":[80],"reasoning).":[81],"this":[83,87],"work,":[84],"we":[85],"apply":[86],"idea":[88,160],"training":[91,110,124,133,137],"of":[92,107,150,157,175],"neural":[93,178],"rankers":[95],"using":[96,185],"curriculum":[97,125],"learning.":[98],"We":[99,112,152],"propose":[100],"several":[101],"heuristics":[102,117],"estimate":[104],"difficulty":[106],"given":[109],"sample.":[111],"show":[113,167],"proposed":[116,159],"can":[118,227],"used":[120],"build":[122],"down-weights":[127],"samples":[129,147],"early":[130],"in":[131,205,211,224],"process.":[134],"As":[135],"process":[138],"progresses,":[139],"our":[140,158,169,197],"approach":[141,170],"gradually":[142],"shifts":[143],"weighting":[145],"equally,":[148],"regardless":[149],"difficulty.":[151],"present":[153],"comprehensive":[155],"evaluation":[156],"on":[161],"three":[162],"datasets.":[165],"Results":[166],"leads":[171],"superior":[173],"performance":[174,230],"two":[176],"leading":[177],"architectures,":[180],"namely":[181],"BERT":[182],"ConvKNRM,":[184],"both":[186],"pointwise":[187],"pairwise":[189],"losses.":[190],"When":[191],"applied":[192],"BERT-based":[195],"ranker,":[196],"method":[198],"yields":[199],"up":[200],"4%":[203],"improvement":[204,210],"MRR":[206],"9%":[209],"[email":[212],"protected]":[213],"(compared":[214],"model":[217],"trained":[218],"without":[219],"curriculum).":[221],"This":[222],"results":[223],"models":[225],"achieve":[228],"comparable":[229],"expensive":[233],"state-of-the-art":[234],"techniques.":[235]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2020-05-13T00:00:00"}