{"id":"https://openalex.org/W2745239372","doi":"https://doi.org/10.1109/icnsc.2017.8000136","title":"Evaluating machine learning algorithms for applications with humans in the loop","display_name":"Evaluating machine learning algorithms for applications with humans in the loop","publication_year":2017,"publication_date":"2017-05-01","ids":{"openalex":"https://openalex.org/W2745239372","doi":"https://doi.org/10.1109/icnsc.2017.8000136","mag":"2745239372"},"language":"en","primary_location":{"id":"doi:10.1109/icnsc.2017.8000136","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icnsc.2017.8000136","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2017 IEEE 14th International Conference on Networking, Sensing and Control (ICNSC)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5022167957","display_name":"Aravind Kota Gopalakrishna","orcid":"https://orcid.org/0000-0001-7810-0659"},"institutions":[{"id":"https://openalex.org/I83019370","display_name":"Eindhoven University of Technology","ror":"https://ror.org/02c2kyt77","country_code":"NL","type":"education","lineage":["https://openalex.org/I83019370"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Aravind Kota Gopalakrishna","raw_affiliation_strings":["Dept. of Mathematics and Computer Science, Eindhoven University of Technology, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dept. of Mathematics and Computer Science, Eindhoven University of Technology, The Netherlands","institution_ids":["https://openalex.org/I83019370"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081886883","display_name":"Tan\u0131r \u00d6z\u00e7elebi","orcid":"https://orcid.org/0000-0001-7092-5998"},"institutions":[{"id":"https://openalex.org/I83019370","display_name":"Eindhoven University of Technology","ror":"https://ror.org/02c2kyt77","country_code":"NL","type":"education","lineage":["https://openalex.org/I83019370"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Tanir Ozcelebi","raw_affiliation_strings":["Dept. of Mathematics and Computer Science, Eindhoven University of Technology, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dept. of Mathematics and Computer Science, Eindhoven University of Technology, The Netherlands","institution_ids":["https://openalex.org/I83019370"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111437816","display_name":"Johan J. Lukkien","orcid":null},"institutions":[{"id":"https://openalex.org/I83019370","display_name":"Eindhoven University of Technology","ror":"https://ror.org/02c2kyt77","country_code":"NL","type":"education","lineage":["https://openalex.org/I83019370"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Johan J. Lukkien","raw_affiliation_strings":["Dept. of Mathematics and Computer Science, Eindhoven University of Technology, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dept. of Mathematics and Computer Science, Eindhoven University of Technology, The Netherlands","institution_ids":["https://openalex.org/I83019370"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5026941307","display_name":"Antonio Liotta","orcid":"https://orcid.org/0000-0002-2773-4421"},"institutions":[{"id":"https://openalex.org/I83019370","display_name":"Eindhoven University of Technology","ror":"https://ror.org/02c2kyt77","country_code":"NL","type":"education","lineage":["https://openalex.org/I83019370"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Antonio Liotta","raw_affiliation_strings":["Electro-Optical Communications, Eindhoven University of Technology, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Electro-Optical Communications, Eindhoven University of Technology, The Netherlands","institution_ids":["https://openalex.org/I83019370"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I83019370"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"7","issue":null,"first_page":"459","last_page":"464"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9883000254631042,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9861000180244446,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7757431864738464},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.7211331725120544},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.69980788230896},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6412690281867981},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6378947496414185},{"id":"https://openalex.org/keywords/performance-metric","display_name":"Performance metric","score":0.6243482828140259},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.5417147278785706},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.507209300994873},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.41793322563171387},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3593413233757019}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7757431864738464},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.7211331725120544},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.69980788230896},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6412690281867981},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6378947496414185},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.6243482828140259},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5417147278785706},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.507209300994873},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.41793322563171387},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3593413233757019},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":8,"locations":[{"id":"doi:10.1109/icnsc.2017.8000136","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icnsc.2017.8000136","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2017 IEEE 14th International Conference on Networking, Sensing and Control (ICNSC)","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.tue.nl:openaire_cris_publications/5f794c31-3c78-46d0-8d6f-c8c1f0ee9e5e","is_oa":false,"landing_page_url":"https://research.tue.nl/en/publications/5f794c31-3c78-46d0-8d6f-c8c1f0ee9e5e","pdf_url":null,"source":{"id":"https://openalex.org/S4406922641","display_name":"TU/e Research Portal","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:873248","is_oa":false,"landing_page_url":"http://library.tue.nl/csp/dare/LinkToRepository.csp?recordnumber=873248","pdf_url":null,"source":{"id":"https://openalex.org/S4406923046","display_name":"TU/e Research Portal (Eindhoven University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"pmh:876294","is_oa":false,"landing_page_url":"http://library.tue.nl/csp/dare/LinkToRepository.csp?recordnumber=876294","pdf_url":null,"source":{"id":"https://openalex.org/S4406923046","display_name":"TU/e Research Portal (Eindhoven University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"pmh:oai:derby.openrepository.com:10545/622672","is_oa":false,"landing_page_url":"http://hdl.handle.net/10545/622672","pdf_url":null,"source":{"id":"https://openalex.org/S4306402138","display_name":"University of Derby Online Research Archive. (University of Derby)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I22128151","host_organization_name":"University of Derby","host_organization_lineage":["https://openalex.org/I22128151"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Meetings and Proceedings"},{"id":"pmh:oai:library.tue.nl:873248","is_oa":false,"landing_page_url":"http://repository.tue.nl/873248","pdf_url":null,"source":{"id":"https://openalex.org/S4406923046","display_name":"TU/e Research Portal (Eindhoven University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"pmh:oai:library.tue.nl:876294","is_oa":false,"landing_page_url":"http://repository.tue.nl/876294","pdf_url":null,"source":{"id":"https://openalex.org/S4406923046","display_name":"TU/e Research Portal (Eindhoven University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"pmh:tue:oai:pure.tue.nl:publications/5f794c31-3c78-46d0-8d6f-c8c1f0ee9e5e","is_oa":false,"landing_page_url":"https://research.tue.nl/nl/publications/5f794c31-3c78-46d0-8d6f-c8c1f0ee9e5e","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proceedings of the IEEE 14th International Conference on Networking, Sensing and Control (ICNSC), 459 - 464","raw_type":"info:eu-repo/semantics/conferencepaper"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W172260869","https://openalex.org/W1481970930","https://openalex.org/W1570448133","https://openalex.org/W1572771420","https://openalex.org/W1881366439","https://openalex.org/W1966771059","https://openalex.org/W2029957774","https://openalex.org/W2033255535","https://openalex.org/W2118075434","https://openalex.org/W2119393863","https://openalex.org/W2119512550","https://openalex.org/W2130695501","https://openalex.org/W2146241755","https://openalex.org/W2146868725","https://openalex.org/W2153409738","https://openalex.org/W2155054776","https://openalex.org/W2162899047","https://openalex.org/W2165766797","https://openalex.org/W2170624716","https://openalex.org/W2338155270","https://openalex.org/W2468725259","https://openalex.org/W3099103017","https://openalex.org/W4244238212","https://openalex.org/W4308326503","https://openalex.org/W6634275188","https://openalex.org/W6677803693","https://openalex.org/W6683944004","https://openalex.org/W6685004102"],"related_works":["https://openalex.org/W4399207312","https://openalex.org/W4361804730","https://openalex.org/W2142113611","https://openalex.org/W2334467465","https://openalex.org/W2018387840","https://openalex.org/W2087870008","https://openalex.org/W2045629210","https://openalex.org/W2162534555","https://openalex.org/W2752178021","https://openalex.org/W2143024819"],"abstract_inverted_index":{"Applications":[0],"employing":[1],"data":[2],"classification":[3,39,176],"such":[4,12],"as":[5,13,95],"smart":[6,160],"lighting":[7,161],"that":[8,89,121,133,171,208],"involve":[9],"human":[10,124],"factors":[11],"perception":[14],"lead":[15],"to":[16,66,110,138,181,190],"non-deterministic":[17,36,164],"input-output":[18,166],"relationships":[19],"where":[20],"more":[21],"than":[22],"one":[23],"output":[24,38,48,107],"may":[25,49],"be":[26,136],"acceptable":[27],"for":[28,56,69],"a":[29,63,70,96,105,153,159],"given":[30,71],"input.":[31],"For":[32],"these":[33],"so":[34,132],"called":[35],"multiple":[37],"(nDMOC)":[40],"problems,":[41],"the":[42,45,57,78,85,101,111,119,129,150,183,191,200,209,220],"relationship":[43],"between":[44],"input":[46],"and":[47,83,114],"change":[50],"over":[51],"time":[52],"making":[53],"it":[54,134],"difficult":[55],"machine":[58],"learning":[59],"(ML)":[60],"algorithms":[61,141,188],"in":[62,92,142],"batch":[64],"setting":[65,145],"make":[67],"predictions":[68],"context.":[72],"In":[73],"this":[74,93],"paper,":[75],"we":[76],"describe":[77],"nature":[79],"of":[80,152,155,175,185,193,216],"nDMOC":[81,194],"problems":[82],"discuss":[84],"Relevance":[86],"Score":[87],"(RS)":[88],"is":[90,108,179,207],"suitable":[91,180],"context":[94,113],"performance":[97,151,184,222],"metric.":[98],"RS":[99,130,173,202,210],"determines":[100],"extent":[102],"by":[103],"which":[104],"predicted":[106],"relevant":[109],"user's":[112],"behaviors,":[115],"taking":[116],"into":[117],"account":[118],"inconsistencies":[120],"come":[122],"with":[123,163,213],"(perception)":[125],"factors.":[126],"We":[127,148],"tailor":[128],"metric":[131],"can":[135],"used":[137],"evaluate":[139],"ML":[140,156,187,198],"an":[143],"online":[144,197],"at":[146],"run-time.":[147],"assess":[149],"number":[154,215],"algorithms,":[157],"using":[158,172],"dataset":[162],"one-to-many":[165],"relationships.":[167],"The":[168],"results":[169],"indicate":[170],"instead":[174],"accuracy":[177],"(CA)":[178],"analyze":[182],"conventional":[186],"applied":[189],"category":[192],"problems.":[195],"Instance-based":[196],"gives":[199],"best":[201],"performance.":[203],"An":[204],"interesting":[205],"finding":[206],"keeps":[211],"increasing":[212,214],"samples,":[217],"even":[218],"after":[219],"CA":[221],"converges.":[223]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":2}],"updated_date":"2026-07-19T07:52:34.831488","created_date":"2025-10-10T00:00:00"}