{"id":"https://openalex.org/W2509810107","doi":"https://doi.org/10.1145/2970276.2970330","title":"Evaluating the evaluations of code recommender systems: a reality check","display_name":"Evaluating the evaluations of code recommender systems: a reality check","publication_year":2016,"publication_date":"2016-08-25","ids":{"openalex":"https://openalex.org/W2509810107","doi":"https://doi.org/10.1145/2970276.2970330","mag":"2509810107"},"language":"en","primary_location":{"id":"doi:10.1145/2970276.2970330","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2970276.2970330","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003806730","display_name":"Sebastian Proksch","orcid":"https://orcid.org/0000-0003-1856-9361"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technical University of Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Sebastian Proksch","raw_affiliation_strings":["TU Darmstadt, Germany"],"affiliations":[{"raw_affiliation_string":"TU Darmstadt, Germany","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047367018","display_name":"Sven Amann","orcid":"https://orcid.org/0000-0003-2952-0475"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technical University of Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sven Amann","raw_affiliation_strings":["TU Darmstadt, Germany"],"affiliations":[{"raw_affiliation_string":"TU Darmstadt, Germany","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054083284","display_name":"Sarah Nadi","orcid":"https://orcid.org/0000-0002-0091-6030"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technical University of Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sarah Nadi","raw_affiliation_strings":["TU Darmstadt, Germany"],"affiliations":[{"raw_affiliation_string":"TU Darmstadt, Germany","institution_ids":["https://openalex.org/I31512782"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078067853","display_name":"Mira Mezini","orcid":"https://orcid.org/0000-0001-6563-7537"},"institutions":[{"id":"https://openalex.org/I31512782","display_name":"Technical University of Darmstadt","ror":"https://ror.org/05n911h24","country_code":"DE","type":"education","lineage":["https://openalex.org/I31512782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Mira Mezini","raw_affiliation_strings":["TU Darmstadt, Germany"],"affiliations":[{"raw_affiliation_string":"TU Darmstadt, Germany","institution_ids":["https://openalex.org/I31512782"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5003806730"],"corresponding_institution_ids":["https://openalex.org/I31512782"],"apc_list":null,"apc_paid":null,"fwci":9.2887,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.97772007,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"111","last_page":"121"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.8803459405899048},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8795076608657837},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.69398033618927},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5910899639129639},{"id":"https://openalex.org/keywords/reality-check","display_name":"Reality check","score":0.58006352186203},{"id":"https://openalex.org/keywords/snippet","display_name":"Snippet","score":0.5619335174560547},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.47623470425605774},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.46591854095458984},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.43785473704338074},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37786951661109924},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.32923227548599243},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.12792906165122986}],"concepts":[{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.8803459405899048},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8795076608657837},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.69398033618927},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5910899639129639},{"id":"https://openalex.org/C2992692520","wikidata":"https://www.wikidata.org/wiki/Q3931124","display_name":"Reality check","level":3,"score":0.58006352186203},{"id":"https://openalex.org/C2777822670","wikidata":"https://www.wikidata.org/wiki/Q1120538","display_name":"Snippet","level":2,"score":0.5619335174560547},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.47623470425605774},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.46591854095458984},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.43785473704338074},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37786951661109924},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32923227548599243},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.12792906165122986},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2970276.2970330","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2970276.2970330","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/12","score":0.4399999976158142,"display_name":"Responsible consumption and production"}],"awards":[{"id":"https://openalex.org/G650486564","display_name":null,"funder_award_id":"01IS12054","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"},{"id":"https://openalex.org/G893031920","display_name":null,"funder_award_id":"SFB #1119, project E1","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320321114","display_name":"Bundesministerium f\u00fcr Bildung und Forschung","ror":"https://ror.org/04pz7b180"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":29,"referenced_works":["https://openalex.org/W6806504","https://openalex.org/W90447038","https://openalex.org/W1970607969","https://openalex.org/W1995650922","https://openalex.org/W2014755981","https://openalex.org/W2021840107","https://openalex.org/W2050187629","https://openalex.org/W2059556545","https://openalex.org/W2062973095","https://openalex.org/W2063528332","https://openalex.org/W2083878868","https://openalex.org/W2102772625","https://openalex.org/W2108632968","https://openalex.org/W2117642462","https://openalex.org/W2123044578","https://openalex.org/W2128042018","https://openalex.org/W2130344546","https://openalex.org/W2142403498","https://openalex.org/W2143861926","https://openalex.org/W2152090421","https://openalex.org/W2153943889","https://openalex.org/W2157976942","https://openalex.org/W2166296138","https://openalex.org/W2166597811","https://openalex.org/W2169952536","https://openalex.org/W2266912522","https://openalex.org/W2405053756","https://openalex.org/W2412916319","https://openalex.org/W4232188856"],"related_works":["https://openalex.org/W1607713096","https://openalex.org/W3036724449","https://openalex.org/W2798329462","https://openalex.org/W1511521437","https://openalex.org/W3201315194","https://openalex.org/W4318980730","https://openalex.org/W2950904665","https://openalex.org/W2985134635","https://openalex.org/W4231058664","https://openalex.org/W2147744656"],"abstract_inverted_index":{"While":[0],"researchers":[1],"develop":[2],"many":[3],"new":[4],"exciting":[5],"code":[6,16],"recommender":[7,89,139],"systems,":[8,140],"such":[9,22,54,96],"as":[10,76],"method-call":[11,88],"completion,":[12,14],"code-snippet":[13],"or":[15],"search,":[17],"an":[18,121],"accurate":[19],"evaluation":[20,102],"of":[21,37,95,138],"systems":[23],"is":[24,125,142],"always":[25],"a":[26,77,86,131],"challenge.":[27],"We":[28,73,91,118],"analyzed":[29],"the":[30,38,51,93,135],"current":[31,39],"literature":[32],"and":[33,80,104],"found":[34],"that":[35,108,120,124],"most":[36],"evaluations":[40,55],"rely":[41],"on":[42,134],"artificial":[43,101,147],"queries":[44,84,98],"extracted":[45],"from":[46,69],"released":[47],"code,":[48],"which":[49],"begs":[50],"question:":[52],"Do":[53],"reflect":[56],"real-life":[57],"usages?":[58],"To":[59],"answer":[60],"this":[61],"question,":[62],"we":[63],"capture":[64],"6,189":[65],"fine-grained":[66],"development":[67],"histories":[68],"real":[70,83,97],"IDE":[71],"interactions.":[72],"use":[74],"them":[75],"ground":[78],"truth":[79],"extract":[81],"7,157":[82],"for":[85],"specific":[87],"system.":[90],"compare":[92],"results":[94],"with":[99],"different":[100],"strategies":[103],"check":[105],"several":[106],"assumptions":[107],"are":[109],"repeatedly":[110],"used":[111],"in":[112,128,146],"research,":[113],"but":[114,141],"never":[115],"empirically":[116],"evaluated.":[117],"find":[119],"evolving":[122],"context":[123],"often":[126],"observed":[127],"practice":[129],"has":[130],"major":[132],"effect":[133],"prediction":[136],"quality":[137],"not":[143],"commonly":[144],"reflected":[145],"evaluations.":[148]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":6},{"year":2017,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}