{"id":"https://openalex.org/W2894621306","doi":"https://doi.org/10.1145/3239573","title":"Reproduce and Improve","display_name":"Reproduce and Improve","publication_year":2018,"publication_date":"2018-09-29","ids":{"openalex":"https://openalex.org/W2894621306","doi":"https://doi.org/10.1145/3239573","mag":"2894621306"},"language":"en","primary_location":{"id":"doi:10.1145/3239573","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3239573","pdf_url":null,"source":{"id":"https://openalex.org/S110189822","display_name":"Journal of Data and Information Quality","issn_l":"1936-1955","issn":["1936-1955","1936-1963"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Quality","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050287516","display_name":"Kevin Roitero","orcid":"https://orcid.org/0000-0002-9191-3280"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Kevin Roitero","raw_affiliation_strings":["University of Udine, Udine, ITA"],"affiliations":[{"raw_affiliation_string":"University of Udine, Udine, ITA","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028918939","display_name":"Michael Soprano","orcid":"https://orcid.org/0000-0002-7337-7592"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Michael Soprano","raw_affiliation_strings":["University of Udine, Udine, ITA"],"affiliations":[{"raw_affiliation_string":"University of Udine, Udine, ITA","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084062073","display_name":"Andrea Brunello","orcid":"https://orcid.org/0000-0003-2063-218X"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Andrea Brunello","raw_affiliation_strings":["University of Udine, Udine, ITA"],"affiliations":[{"raw_affiliation_string":"University of Udine, Udine, ITA","institution_ids":["https://openalex.org/I129043915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5005575421","display_name":"Stefano Mizzaro","orcid":"https://orcid.org/0000-0002-2852-168X"},"institutions":[{"id":"https://openalex.org/I129043915","display_name":"University of Udine","ror":"https://ror.org/05ht0mh31","country_code":"IT","type":"education","lineage":["https://openalex.org/I129043915"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Stefano Mizzaro","raw_affiliation_strings":["University of Udine, Udine, ITA"],"affiliations":[{"raw_affiliation_string":"University of Udine, Udine, ITA","institution_ids":["https://openalex.org/I129043915"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5050287516"],"corresponding_institution_ids":["https://openalex.org/I129043915"],"apc_list":null,"apc_paid":null,"fwci":2.7488,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.92637729,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"10","issue":"3","first_page":"1","last_page":"21"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.988099992275238,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9871000051498413,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.877906322479248},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6635348200798035},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.625912070274353},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.624388575553894},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5364744067192078},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.5131528973579407},{"id":"https://openalex.org/keywords/reduction","display_name":"Reduction (mathematics)","score":0.4884149432182312},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.463454931974411},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4475204348564148},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4277043640613556},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37405937910079956},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3349241614341736}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.877906322479248},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6635348200798035},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.625912070274353},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.624388575553894},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5364744067192078},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.5131528973579407},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.4884149432182312},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.463454931974411},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4475204348564148},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4277043640613556},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37405937910079956},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3349241614341736},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3239573","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3239573","pdf_url":null,"source":{"id":"https://openalex.org/S110189822","display_name":"Journal of Data and Information Quality","issn_l":"1936-1955","issn":["1936-1955","1936-1963"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Data and Information Quality","raw_type":"journal-article"},{"id":"pmh:oai:air.uniud.it:11390/1140167","is_oa":false,"landing_page_url":"https://dl.acm.org/citation.cfm?id=3239573","pdf_url":null,"source":{"id":"https://openalex.org/S4306401163","display_name":"Institutional Research Information System (University of Udine)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I129043915","host_organization_name":"University of Udine","host_organization_lineage":["https://openalex.org/I129043915"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"internazionale"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W159389352","https://openalex.org/W1512383952","https://openalex.org/W1574528120","https://openalex.org/W1579063438","https://openalex.org/W1824124613","https://openalex.org/W1976076261","https://openalex.org/W1980161923","https://openalex.org/W1983595289","https://openalex.org/W1987690005","https://openalex.org/W1994960885","https://openalex.org/W2023217403","https://openalex.org/W2027359109","https://openalex.org/W2073722401","https://openalex.org/W2075893676","https://openalex.org/W2083622154","https://openalex.org/W2091524854","https://openalex.org/W2100240666","https://openalex.org/W2126105956","https://openalex.org/W2138621811","https://openalex.org/W2154884126","https://openalex.org/W2468776465","https://openalex.org/W2469056188","https://openalex.org/W2570788028","https://openalex.org/W2605232020","https://openalex.org/W4233760227"],"related_works":["https://openalex.org/W2012531322","https://openalex.org/W2785900585","https://openalex.org/W2353730437","https://openalex.org/W2490303674","https://openalex.org/W2609066826","https://openalex.org/W2810752900","https://openalex.org/W3186538219","https://openalex.org/W2365677836","https://openalex.org/W2531295127","https://openalex.org/W2389015757"],"abstract_inverted_index":{"Effectiveness":[0],"evaluation":[1,233],"of":[2,8,24,69,84,98,119,148,197,231],"information":[3],"retrieval":[4],"systems":[5,106],"by":[6],"means":[7],"a":[9,13,36,67,99,134,141],"test":[10],"collection":[11],"is":[12,19,49,60,180],"widely":[14],"used":[15,204],"methodology.":[16],"However,":[17],"it":[18,89],"rather":[20],"expensive":[21],"in":[22,46,54,107],"terms":[23],"resources,":[25],"time,":[26],"and":[27,136,171,189,201],"money;":[28],"therefore,":[29],"many":[30],"researchers":[31],"have":[32,79,214],"proposed":[33,92],"methods":[34],"for":[35,205],"cheaper":[37],"evaluation.":[38,207],"One":[39],"particular":[40],"approach,":[41],"on":[42,66,140,159],"which":[43,213],"we":[44,218],"focus":[45],"this":[47],"article,":[48],"to":[50,93,112,154,169,182,190,193,222],"use":[51],"fewer":[52],"topics:":[53],"TREC-like":[55],"initiatives,":[56],"usually":[57],"system":[58],"effectiveness":[59,65],"evaluated":[61],"as":[62,163,165],"the":[63,86,95,105,108,113,120,126,185,195,198,206],"average":[64],"set":[68,161],"n":[70,73],"topics":[71,102],"(usually,":[72],"=50,":[74],"but":[75],"more":[76],"than":[77],"1,000":[78],"been":[80,91,130,215],"also":[81,220],"adopted);":[82],"instead":[83],"using":[85],"full":[87,114],"set,":[88],"has":[90,122,129],"find":[94],"best":[96],"subsets":[97],"few":[100],"good":[101],"that":[103,128,177],"evaluate":[104],"most":[109],"similar":[110],"way":[111],"set.":[115],"The":[116,145],"computational":[117],"complexity":[118],"task":[121],"so":[123],"far":[124],"limited":[125],"analysis":[127],"performed.":[131],"We":[132,175],"develop":[133],"novel":[135],"efficient":[137],"approach":[138,179],"based":[139],"multi-objective":[142],"evolutionary":[143],"algorithm.":[144],"higher":[146],"efficiency":[147],"our":[149,178],"new":[150,167],"implementation":[151],"allows":[152],"us":[153,192],"reproduce":[155,184],"some":[156,224],"notable":[157],"results":[158,188,230],"topic":[160,226],"reduction,":[162],"well":[164],"perform":[166],"experiments":[168],"generalize":[170],"improve":[172],"such":[173],"results.":[174],"show":[176],"able":[181,221],"both":[183],"main":[186],"state-of-the-art":[187],"allow":[191],"analyze":[194],"effect":[196],"collection,":[199],"metric,":[200],"pool":[202],"depth":[203],"Finally,":[208],"differently":[209],"from":[210],"previous":[211],"studies,":[212],"mainly":[216],"theoretical,":[217],"are":[219],"discuss":[223],"practical":[225],"selection":[227],"strategies,":[228],"integrating":[229],"automatic":[232],"approaches.":[234]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":5}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}