{"id":"https://openalex.org/W4391760813","doi":"https://doi.org/10.1145/3632754.3633480","title":"Efficiency of Large Language Models to scale up Ground Truth: Overview of the IRSE Track at Forum for Information Retrieval 2023","display_name":"Efficiency of Large Language Models to scale up Ground Truth: Overview of the IRSE Track at Forum for Information Retrieval 2023","publication_year":2023,"publication_date":"2023-12-15","ids":{"openalex":"https://openalex.org/W4391760813","doi":"https://doi.org/10.1145/3632754.3633480"},"language":"en","primary_location":{"id":"doi:10.1145/3632754.3633480","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3632754.3633480","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3632754.3633480","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th Annual Meeting of the Forum for Information Retrieval Evaluation","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3632754.3633480","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013058646","display_name":"Soumen Paul","orcid":"https://orcid.org/0000-0003-3548-2663"},"institutions":[{"id":"https://openalex.org/I145894827","display_name":"Indian Institute of Technology Kharagpur","ror":"https://ror.org/03w5sq511","country_code":"IN","type":"education","lineage":["https://openalex.org/I145894827"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Soumen Paul","raw_affiliation_strings":["Department of Computer Science and Engineering, Indian Institute of Technology Kharagpur, India"],"raw_orcid":"https://orcid.org/0000-0003-3548-2663","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, Indian Institute of Technology Kharagpur, India","institution_ids":["https://openalex.org/I145894827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086875394","display_name":"Srijoni Majumdar","orcid":"https://orcid.org/0000-0003-3935-4087"},"institutions":[{"id":"https://openalex.org/I130828816","display_name":"University of Leeds","ror":"https://ror.org/024mrxd33","country_code":"GB","type":"education","lineage":["https://openalex.org/I130828816"]},{"id":"https://openalex.org/I145894827","display_name":"Indian Institute of Technology Kharagpur","ror":"https://ror.org/03w5sq511","country_code":"IN","type":"education","lineage":["https://openalex.org/I145894827"]}],"countries":["GB","IN"],"is_corresponding":false,"raw_author_name":"Srijoni Majumdar","raw_affiliation_strings":["Indian Institute of Technology Kharagpur, India and University of Leeds, UK"],"raw_orcid":"https://orcid.org/0000-0003-3935-4087","affiliations":[{"raw_affiliation_string":"Indian Institute of Technology Kharagpur, India and University of Leeds, UK","institution_ids":["https://openalex.org/I130828816","https://openalex.org/I145894827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110280323","display_name":"Ayan Bandyopadhyay","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ayan Bandyopadhyay","raw_affiliation_strings":["TCG CREST, India"],"raw_orcid":"https://orcid.org/0000-0002-4700-6456","affiliations":[{"raw_affiliation_string":"TCG CREST, India","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012548633","display_name":"Bhargav Dave","orcid":"https://orcid.org/0000-0003-2742-480X"},"institutions":[{"id":"https://openalex.org/I98389781","display_name":"Dhirubhai Ambani University","ror":"https://ror.org/02d5b7g69","country_code":"IN","type":"education","lineage":["https://openalex.org/I98389781"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Bhargav Dave","raw_affiliation_strings":["DA-IICT, India"],"raw_orcid":"https://orcid.org/0000-0003-2742-480X","affiliations":[{"raw_affiliation_string":"DA-IICT, India","institution_ids":["https://openalex.org/I98389781"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075720299","display_name":"Samiran Chattopadhyay","orcid":"https://orcid.org/0000-0002-8929-9605"},"institutions":[{"id":"https://openalex.org/I170979836","display_name":"Jadavpur University","ror":"https://ror.org/02af4h012","country_code":"IN","type":"education","lineage":["https://openalex.org/I170979836"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Samiran Chattopadhyay","raw_affiliation_strings":["Jadavpur University, India"],"raw_orcid":"https://orcid.org/0000-0002-8929-9605","affiliations":[{"raw_affiliation_string":"Jadavpur University, India","institution_ids":["https://openalex.org/I170979836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011440310","display_name":"Partha Pratim Das","orcid":"https://orcid.org/0000-0003-1435-6051"},"institutions":[{"id":"https://openalex.org/I145894827","display_name":"Indian Institute of Technology Kharagpur","ror":"https://ror.org/03w5sq511","country_code":"IN","type":"education","lineage":["https://openalex.org/I145894827"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Partha Das","raw_affiliation_strings":["Indian Institute of Technology Kharagpur, India"],"raw_orcid":"https://orcid.org/0000-0003-1435-6051","affiliations":[{"raw_affiliation_string":"Indian Institute of Technology Kharagpur, India","institution_ids":["https://openalex.org/I145894827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011889540","display_name":"Paul Clough","orcid":"https://orcid.org/0000-0003-1739-175X"},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Paul D Clough","raw_affiliation_strings":["University of Sheffield, UK and TPXimpact London, UK"],"raw_orcid":"https://orcid.org/0000-0003-1739-175X","affiliations":[{"raw_affiliation_string":"University of Sheffield, UK and TPXimpact London, UK","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5026950398","display_name":"Prasenjit Majumder","orcid":"https://orcid.org/0000-0003-0840-9313"},"institutions":[{"id":"https://openalex.org/I98389781","display_name":"Dhirubhai Ambani University","ror":"https://ror.org/02d5b7g69","country_code":"IN","type":"education","lineage":["https://openalex.org/I98389781"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Prasenjit Majumder","raw_affiliation_strings":["TCG CREST, India and DA-IICT, India"],"raw_orcid":"https://orcid.org/0000-0003-0840-9313","affiliations":[{"raw_affiliation_string":"TCG CREST, India and DA-IICT, India","institution_ids":["https://openalex.org/I98389781"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.631,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.71309582,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"16","last_page":"18"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9731000065803528,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.972100019454956,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8129062056541443},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.5838031768798828},{"id":"https://openalex.org/keywords/track","display_name":"Track (disk drive)","score":0.5818547010421753},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5790884494781494},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5432563424110413},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5068884491920471},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.49312326312065125},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.49292683601379395},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.49253758788108826},{"id":"https://openalex.org/keywords/hyperparameter","display_name":"Hyperparameter","score":0.4887351989746094},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.48451846837997437},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.48395588994026184},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.45519813895225525},{"id":"https://openalex.org/keywords/binary-classification","display_name":"Binary classification","score":0.4299355745315552},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4284172058105469},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"Sentiment analysis","score":0.41285330057144165},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3553121089935303},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3350730836391449},{"id":"https://openalex.org/keywords/support-vector-machine","display_name":"Support vector machine","score":0.1342327892780304},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1240113377571106}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8129062056541443},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.5838031768798828},{"id":"https://openalex.org/C89992363","wikidata":"https://www.wikidata.org/wiki/Q5961558","display_name":"Track (disk drive)","level":2,"score":0.5818547010421753},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5790884494781494},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5432563424110413},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5068884491920471},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.49312326312065125},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.49292683601379395},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.49253758788108826},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.4887351989746094},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48451846837997437},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48395588994026184},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.45519813895225525},{"id":"https://openalex.org/C66905080","wikidata":"https://www.wikidata.org/wiki/Q17005494","display_name":"Binary classification","level":3,"score":0.4299355745315552},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4284172058105469},{"id":"https://openalex.org/C66402592","wikidata":"https://www.wikidata.org/wiki/Q2271421","display_name":"Sentiment analysis","level":2,"score":0.41285330057144165},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3553121089935303},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3350730836391449},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.1342327892780304},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1240113377571106},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3632754.3633480","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3632754.3633480","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3632754.3633480","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th Annual Meeting of the Forum for Information Retrieval Evaluation","raw_type":"proceedings-article"},{"id":"pmh:oai:eprints.whiterose.ac.uk:237532","is_oa":false,"landing_page_url":"https://orcid.org/0000-0003-3935-4087>,","pdf_url":null,"source":{"id":"https://openalex.org/S4306400854","display_name":"White Rose Research Online (University of Leeds, The University of Sheffield, University of York)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2800616092","host_organization_name":"White Rose University Consortium","host_organization_lineage":["https://openalex.org/I2800616092"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Conference abstract"}],"best_oa_location":{"id":"doi:10.1145/3632754.3633480","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3632754.3633480","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3632754.3633480","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 15th Annual Meeting of the Forum for Information Retrieval Evaluation","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.4699999988079071,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391760813.pdf","grobid_xml":"https://content.openalex.org/works/W4391760813.grobid-xml"},"referenced_works_count":8,"referenced_works":["https://openalex.org/W2529336739","https://openalex.org/W2979284645","https://openalex.org/W3158309582","https://openalex.org/W4281755696","https://openalex.org/W4285081531","https://openalex.org/W4320028630","https://openalex.org/W4328028774","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W4390421286","https://openalex.org/W4280563792","https://openalex.org/W2140186469","https://openalex.org/W4389724018","https://openalex.org/W4318719684","https://openalex.org/W2775233965","https://openalex.org/W4318559728","https://openalex.org/W4360995913","https://openalex.org/W4312193868","https://openalex.org/W4308165509"],"abstract_inverted_index":{"The":[0,50],"Software":[1],"Engineering":[2],"Information":[3],"Retrieval":[4],"(IRSE)":[5],"track":[6],"aims":[7],"to":[8,140],"devise":[9],"solutions":[10],"for":[11],"the":[12,102,109,112,135],"automated":[13],"evaluation":[14],"of":[15,55],"code":[16,56,60],"comments":[17,43,57],"within":[18],"a":[19,38],"machine":[20],"learning":[21,114],"framework,":[22],"with":[23],"labels":[24,126],"generated":[25,73,127],"by":[26,74,128],"both":[27],"humans":[28],"and":[29,58,69,87,104,117],"large":[30,77,129],"language":[31,78,130],"models.":[32,79],"Within":[33],"this":[34],"track,":[35],"there":[36],"is":[37,122],"binary":[39],"classification":[40],"task:":[41],"discerning":[42],"as":[44],"either":[45],"useful":[46],"or":[47],"not":[48],"useful.":[49],"dataset":[51,72],"includes":[52],"9,048":[53],"pairs":[54],"surrounding":[59],"snippets":[61],"drawn":[62],"from":[63],"open-source":[64],"C-based":[65],"projects":[66],"on":[67,108],"GitHub":[68],"an":[70],"additional":[71],"teams":[75,83],"employing":[76],"In":[80],"total,":[81],"17":[82],"representing":[84],"various":[85],"universities":[86],"software":[88],"companies":[89],"have":[90],"contributed":[91],"56":[92],"experiments.":[93],"These":[94],"experiments":[95],"were":[96],"assessed":[97],"through":[98],"quantitative":[99],"metrics,":[100],"primarily":[101],"F1-Score,":[103],"qualitative":[105],"evaluations":[106],"based":[107],"features":[110],"developed,":[111],"supervised":[113],"models":[115,131],"employed,":[116],"their":[118],"respective":[119],"hyperparameters.":[120],"It":[121],"worth":[123],"noting":[124],"that":[125],"introduce":[132],"bias":[133],"into":[134],"prediction":[136],"model":[137],"but":[138],"lead":[139],"less":[141],"over-fitted":[142],"results.":[143]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-08-25T07:29:55.448023","created_date":"2025-10-10T00:00:00"}