{"id":"https://openalex.org/W3152887675","doi":"https://doi.org/10.1145/3404835.3463262","title":"How Deep is your Learning: the DL-HARD Annotated Deep Learning Dataset","display_name":"How Deep is your Learning: the DL-HARD Annotated Deep Learning Dataset","publication_year":2021,"publication_date":"2021-07-11","ids":{"openalex":"https://openalex.org/W3152887675","doi":"https://doi.org/10.1145/3404835.3463262","mag":"3152887675"},"language":"en","primary_location":{"id":"doi:10.1145/3404835.3463262","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404835.3463262","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://eprints.gla.ac.uk/239176/2/239176.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5085187522","display_name":"Iain Mackie","orcid":"https://orcid.org/0000-0002-9690-9854"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Iain Mackie","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071842569","display_name":"Jeff Dalton","orcid":"https://orcid.org/0000-0003-2422-8651"},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jeffrey Dalton","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5059489981","display_name":"Andrew Yates","orcid":"https://orcid.org/0000-0002-5970-880X"},"institutions":[{"id":"https://openalex.org/I4210109712","display_name":"Max Planck Institute for Informatics","ror":"https://ror.org/01w19ak89","country_code":"DE","type":"facility","lineage":["https://openalex.org/I149899117","https://openalex.org/I4210109712"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Andrew Yates","raw_affiliation_strings":["Max Planck Institute for Informatics, Saarbr\u00fccken, Germany"],"affiliations":[{"raw_affiliation_string":"Max Planck Institute for Informatics, Saarbr\u00fccken, Germany","institution_ids":["https://openalex.org/I4210109712"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5085187522"],"corresponding_institution_ids":["https://openalex.org/I7882870"],"apc_list":null,"apc_paid":null,"fwci":3.3591,"has_fulltext":true,"cited_by_count":41,"citation_normalized_percentile":{"value":0.93351237,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"2335","last_page":"2341"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7812708616256714},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.7482256889343262},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7241932153701782},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.7014962434768677},{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.6740071177482605},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6348897814750671},{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"Learning to rank","score":0.5480034351348877},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5475755929946899},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5040430426597595},{"id":"https://openalex.org/keywords/resource","display_name":"Resource (disambiguation)","score":0.4486824572086334},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2485201358795166}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7812708616256714},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7482256889343262},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7241932153701782},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.7014962434768677},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.6740071177482605},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6348897814750671},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.5480034351348877},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5475755929946899},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5040430426597595},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.4486824572086334},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2485201358795166},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3404835.3463262","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404835.3463262","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:dare.uva.nl:openaire_cris_publications/6ada8dec-f3d7-4fd2-b9d7-0826a857a44a","is_oa":false,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/how-deep-is-your-learning-the-dlhard-annotated-deep-learning-dataset(6ada8dec-f3d7-4fd2-b9d7-0826a857a44a).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Mackie, I, Dalton, J & Yates, A 2021, How Deep is your Learning: the DL-HARD Annotated Deep Learning Dataset. in SIGIR '21 : proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval : July 11-15, 2021, virtual event, Canada. Association for Computing Machinery, New York, NY, pp. 2335\u20132341, 44th International ACM SIGIR Conference on Research and Development in Information Retrieval, Montr\u00e9al, Quebec, Canada, 11/07/21. https://doi.org/10.1145/3404835.3463262","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:eprints.gla.ac.uk:239176","is_oa":true,"landing_page_url":"https://eprints.gla.ac.uk/view/author/54266.html>,","pdf_url":"https://eprints.gla.ac.uk/239176/2/239176.pdf","source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"}],"best_oa_location":{"id":"pmh:oai:eprints.gla.ac.uk:239176","is_oa":true,"landing_page_url":"https://eprints.gla.ac.uk/view/author/54266.html>,","pdf_url":"https://eprints.gla.ac.uk/239176/2/239176.pdf","source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"PeerReviewed"},"sustainable_development_goals":[{"score":0.4399999976158142,"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals"}],"awards":[{"id":"https://openalex.org/G1934935867","display_name":null,"funder_award_id":"Engineering and Physical Sciences R","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G3203412108","display_name":"Turing AI Fellowship:Neural Conversational Information Seeking Assistant","funder_award_id":"EP/V025708/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"},{"id":"https://openalex.org/G7302224536","display_name":null,"funder_award_id":"EP/V025708/1","funder_id":"https://openalex.org/F4320334627","funder_display_name":"Engineering and Physical Sciences Research Council"}],"funders":[{"id":"https://openalex.org/F4320334627","display_name":"Engineering and Physical Sciences Research Council","ror":"https://ror.org/0439y7842"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3152887675.pdf","grobid_xml":"https://content.openalex.org/works/W3152887675.grobid-xml"},"referenced_works_count":17,"referenced_works":["https://openalex.org/W2000246295","https://openalex.org/W2000411838","https://openalex.org/W2252136820","https://openalex.org/W2905300009","https://openalex.org/W2963748441","https://openalex.org/W2964221236","https://openalex.org/W2981852735","https://openalex.org/W3013372641","https://openalex.org/W3033919759","https://openalex.org/W3039677769","https://openalex.org/W3098350697","https://openalex.org/W3100107515","https://openalex.org/W3104748221","https://openalex.org/W3134455255","https://openalex.org/W3134665270","https://openalex.org/W4288089799","https://openalex.org/W6766496051"],"related_works":["https://openalex.org/W3127142483","https://openalex.org/W4385565564","https://openalex.org/W2138488530","https://openalex.org/W2898073868","https://openalex.org/W2971071571","https://openalex.org/W2798835721","https://openalex.org/W2387658907","https://openalex.org/W2922169395","https://openalex.org/W2385796165","https://openalex.org/W25098770"],"abstract_inverted_index":{"Deep":[0,24],"Learning":[1,25],"Hard":[2],"(DL-HARD)":[3],"is":[4,109],"a":[5,47,58,110],"new":[6,111],"annotated":[7],"dataset":[8],"designed":[9],"to":[10,91],"more":[11],"effectively":[12],"evaluate":[13],"neural":[14,117],"ranking":[15,103,118],"models":[16],"on":[17,22,53,93,116,122],"complex":[18,125],"topics.":[19,126],"It":[20],"builds":[21],"TREC":[23],"(DL)":[26],"topics":[27,67],"by":[28,120],"extensively":[29],"annotating":[30],"them":[31],"with":[32],"question":[33],"intent":[34],"categories,":[35,41],"answer":[36],"types,":[37],"wikified":[38],"entities,":[39],"topic":[40],"and":[42,80,95,101,124],"result":[43],"type":[44],"metadata":[45],"from":[46,68],"commercial":[48],"web":[49],"search":[50],"engine.":[51],"Based":[52],"this":[54],"data,":[55],"we":[56],"introduce":[57],"framework":[59],"for":[60],"identifying":[61],"challenging":[62,123],"queries.":[63],"DL-HARD":[64,94,108],"contains":[65],"fifty":[66],"the":[69,87,102],"official":[70,88],"DL":[71,92],"2019/2020":[72],"evaluation":[73],"benchmark,":[74],"half":[75],"of":[76,104],"which":[77],"are":[78],"newly":[79],"independently":[81],"assessed.":[82],"We":[83],"perform":[84],"experiments":[85],"using":[86],"submitted":[89],"runs":[90],"find":[96],"substantial":[97],"differences":[98],"in":[99],"metrics":[100],"participating":[105],"systems.":[106],"Overall,":[107],"resource":[112],"that":[113],"promotes":[114],"research":[115],"methods":[119],"focusing":[121]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}