{"id":"https://openalex.org/W4408894417","doi":"https://doi.org/10.1145/3719159.3721223","title":"Towards A Modular End-To-End Machine Learning Benchmarking Framework","display_name":"Towards A Modular End-To-End Machine Learning Benchmarking Framework","publication_year":2025,"publication_date":"2025-03-27","ids":{"openalex":"https://openalex.org/W4408894417","doi":"https://doi.org/10.1145/3719159.3721223"},"language":"en","primary_location":{"id":"doi:10.1145/3719159.3721223","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3719159.3721223","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd International Workshop on Testing Distributed Internet of Things Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3719159.3721223","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063646475","display_name":"Robert Bayer","orcid":"https://orcid.org/0000-0002-1052-3855"},"institutions":[{"id":"https://openalex.org/I83467386","display_name":"IT University of Copenhagen","ror":"https://ror.org/02309jg23","country_code":"DK","type":"education","lineage":["https://openalex.org/I83467386"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Robert Bayer","raw_affiliation_strings":["IT University of Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0000-0002-1052-3855","affiliations":[{"raw_affiliation_string":"IT University of Copenhagen, Denmark","institution_ids":["https://openalex.org/I83467386"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006238663","display_name":"Ties Robroek","orcid":"https://orcid.org/0009-0006-3451-5602"},"institutions":[{"id":"https://openalex.org/I83467386","display_name":"IT University of Copenhagen","ror":"https://ror.org/02309jg23","country_code":"DK","type":"education","lineage":["https://openalex.org/I83467386"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Ties Robroek","raw_affiliation_strings":["IT University of Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0009-0006-3451-5602","affiliations":[{"raw_affiliation_string":"IT University of Copenhagen, Denmark","institution_ids":["https://openalex.org/I83467386"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061990490","display_name":"P\u0131nar T\u00f6z\u00fcn","orcid":"https://orcid.org/0000-0001-6838-4854"},"institutions":[{"id":"https://openalex.org/I83467386","display_name":"IT University of Copenhagen","ror":"https://ror.org/02309jg23","country_code":"DK","type":"education","lineage":["https://openalex.org/I83467386"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Pinar T\u00f6z\u00fcn","raw_affiliation_strings":["IT University of Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0000-0001-6838-4854","affiliations":[{"raw_affiliation_string":"IT University of Copenhagen, Denmark","institution_ids":["https://openalex.org/I83467386"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.5175,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.92083474,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"23","last_page":"26"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9854999780654907,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9799000024795532,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.7286555171012878},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6846048831939697},{"id":"https://openalex.org/keywords/modular-design","display_name":"Modular design","score":0.6753500699996948},{"id":"https://openalex.org/keywords/end-to-end-principle","display_name":"End-to-end principle","score":0.6481362581253052},{"id":"https://openalex.org/keywords/end-user-development","display_name":"End-user development","score":0.4775277376174927},{"id":"https://openalex.org/keywords/end-user","display_name":"End user","score":0.4116140604019165},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.283035546541214},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.2695131301879883},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.08472013473510742}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7286555171012878},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6846048831939697},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.6753500699996948},{"id":"https://openalex.org/C74296488","wikidata":"https://www.wikidata.org/wiki/Q2527392","display_name":"End-to-end principle","level":2,"score":0.6481362581253052},{"id":"https://openalex.org/C2776867947","wikidata":"https://www.wikidata.org/wiki/Q500467","display_name":"End-user development","level":3,"score":0.4775277376174927},{"id":"https://openalex.org/C91262260","wikidata":"https://www.wikidata.org/wiki/Q528074","display_name":"End user","level":2,"score":0.4116140604019165},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.283035546541214},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2695131301879883},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.08472013473510742},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3719159.3721223","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3719159.3721223","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd International Workshop on Testing Distributed Internet of Things Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.atira.dk:openaire/7702f384-2591-4960-abce-f443e9eac67d","is_oa":true,"landing_page_url":"https://pure.itu.dk/portal/da/publications/7702f384-2591-4960-abce-f443e9eac67d","pdf_url":null,"source":{"id":"https://openalex.org/S4377196680","display_name":"IT University Of Copenhagen (IT University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I83467386","host_organization_name":"IT University of Copenhagen","host_organization_lineage":["https://openalex.org/I83467386"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Bayer, R, Robroek, T T & T\u00f6z\u00fcn, P 2025, 'Towards A Modular End-To-End Machine Learning Benchmarking Framework', TDIS '25, pp. 23\u201326. https://doi.org/10.1145/3719159.3721223","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"doi:10.1145/3719159.3721223","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3719159.3721223","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 3rd International Workshop on Testing Distributed Internet of Things Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G8244615741","display_name":null,"funder_award_id":"NNF22OC0079398","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"}],"funders":[{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W2099001231","https://openalex.org/W2751343396","https://openalex.org/W2909837330","https://openalex.org/W2970139027","https://openalex.org/W2990138404","https://openalex.org/W4360998026","https://openalex.org/W4368353224","https://openalex.org/W4380925477","https://openalex.org/W4386768654","https://openalex.org/W4394944658","https://openalex.org/W4394969374","https://openalex.org/W4404181195"],"related_works":["https://openalex.org/W4299590256","https://openalex.org/W3163634122","https://openalex.org/W2113114423","https://openalex.org/W4287122814","https://openalex.org/W4391709660","https://openalex.org/W2529145526","https://openalex.org/W2884814595","https://openalex.org/W4327633437","https://openalex.org/W4396816926","https://openalex.org/W4393212338"],"abstract_inverted_index":{"Machine":[0],"learning":[1],"(ML)":[2],"benchmarks":[3],"are":[4,66],"crucial":[5],"for":[6,68],"evaluating":[7],"the":[8,18],"performance,":[9],"efficiency,":[10],"and":[11,71],"scalability":[12],"of":[13,20],"ML":[14,22],"systems,":[15],"especially":[16],"as":[17,25],"adoption":[19],"complex":[21],"pipelines,":[23],"such":[24],"retrieval-augmented":[26],"generation":[27],"(RAG),":[28],"continues":[29],"to":[30,81],"grow.":[31],"These":[32],"pipelines":[33],"introduce":[34,53],"intricate":[35],"execution":[36],"graphs":[37],"that":[38,56],"require":[39],"more":[40],"advanced":[41],"benchmarking":[42,77],"approaches.":[43],"Additionally,":[44],"collocating":[45],"workloads":[46],"can":[47],"improve":[48],"resource":[49,64],"efficiency":[50],"but":[51],"may":[52],"contention":[54],"challenges":[55],"must":[57],"be":[58],"carefully":[59],"managed.":[60],"Detailed":[61],"insights":[62],"into":[63],"utilization":[65],"necessary":[67],"effective":[69],"collocation":[70],"optimized":[72],"edge":[73],"deployments.":[74],"However,":[75],"existing":[76],"frameworks":[78],"often":[79],"fail":[80],"capture":[82],"these":[83],"critical":[84],"aspects.":[85]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}