{"id":"https://openalex.org/W4392402503","doi":"https://doi.org/10.5220/0012462000003660","title":"Reliability and Stability of Mean Opinion Score for Image Aesthetic Quality Assessment Obtained Through Crowdsourcing","display_name":"Reliability and Stability of Mean Opinion Score for Image Aesthetic Quality Assessment Obtained Through Crowdsourcing","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4392402503","doi":"https://doi.org/10.5220/0012462000003660"},"language":"en","primary_location":{"id":"doi:10.5220/0012462000003660","is_oa":true,"landing_page_url":"https://doi.org/10.5220/0012462000003660","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 19th International Joint Conference on Computer Vision, Imaging and Computer Graphics Theory and Applications","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.5220/0012462000003660","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076904517","display_name":"Egor Ershov","orcid":"https://orcid.org/0000-0001-6797-6284"},"institutions":[{"id":"https://openalex.org/I4210107660","display_name":"Institute for Information Transmission Problems","ror":"https://ror.org/013w2d378","country_code":"RU","type":"facility","lineage":["https://openalex.org/I1313323035","https://openalex.org/I4210097085","https://openalex.org/I4210107660"]}],"countries":["RU"],"is_corresponding":true,"raw_author_name":"Egor Ershov","raw_affiliation_strings":["Institute for Information Transmission Problems, Russian Academy of Sciences, 119991 Moscow, Russia, --- Select a Country ---"],"affiliations":[{"raw_affiliation_string":"Institute for Information Transmission Problems, Russian Academy of Sciences, 119991 Moscow, Russia, --- Select a Country ---","institution_ids":["https://openalex.org/I4210107660"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094056117","display_name":"Artyom Panshin","orcid":"https://orcid.org/0009-0009-0038-4888"},"institutions":[{"id":"https://openalex.org/I4210107660","display_name":"Institute for Information Transmission Problems","ror":"https://ror.org/013w2d378","country_code":"RU","type":"facility","lineage":["https://openalex.org/I1313323035","https://openalex.org/I4210097085","https://openalex.org/I4210107660"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Artyom Panshin","raw_affiliation_strings":["Institute for Information Transmission Problems, Russian Academy of Sciences, 119991 Moscow, Russia, --- Select a Country ---"],"affiliations":[{"raw_affiliation_string":"Institute for Information Transmission Problems, Russian Academy of Sciences, 119991 Moscow, Russia, --- Select a Country ---","institution_ids":["https://openalex.org/I4210107660"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092636029","display_name":"Ivan Ermakov","orcid":null},"institutions":[{"id":"https://openalex.org/I4210107660","display_name":"Institute for Information Transmission Problems","ror":"https://ror.org/013w2d378","country_code":"RU","type":"facility","lineage":["https://openalex.org/I1313323035","https://openalex.org/I4210097085","https://openalex.org/I4210107660"]}],"countries":["RU"],"is_corresponding":false,"raw_author_name":"Ivan Ermakov","raw_affiliation_strings":["Institute for Information Transmission Problems, Russian Academy of Sciences, 119991 Moscow, Russia, --- Select a Country ---"],"affiliations":[{"raw_affiliation_string":"Institute for Information Transmission Problems, Russian Academy of Sciences, 119991 Moscow, Russia, --- Select a Country ---","institution_ids":["https://openalex.org/I4210107660"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053645603","display_name":"Nikola Bani\u0107","orcid":"https://orcid.org/0000-0002-3900-8590"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nikola Bani\u0107","raw_affiliation_strings":["Gideon Brothers, 10000 Zagreb, Croatia, --- Select a Country ---"],"affiliations":[{"raw_affiliation_string":"Gideon Brothers, 10000 Zagreb, Croatia, --- Select a Country ---","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051694844","display_name":"Alexey Savchik","orcid":"https://orcid.org/0000-0003-3035-1365"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alex Savchik","raw_affiliation_strings":["ACMetric, Netherlands, --- Select a Country ---"],"affiliations":[{"raw_affiliation_string":"ACMetric, Netherlands, --- Select a Country ---","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013570285","display_name":"Simone Bianco","orcid":"https://orcid.org/0000-0002-7070-1545"},"institutions":[{"id":"https://openalex.org/I66752286","display_name":"University of Milano-Bicocca","ror":"https://ror.org/01ynf4891","country_code":"IT","type":"education","lineage":["https://openalex.org/I66752286"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Simone Bianco","raw_affiliation_strings":["University of Milano-Bicocca, 20126 Milan, Italy, --- Select a Country ---"],"affiliations":[{"raw_affiliation_string":"University of Milano-Bicocca, 20126 Milan, Italy, --- Select a Country ---","institution_ids":["https://openalex.org/I66752286"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5076904517"],"corresponding_institution_ids":["https://openalex.org/I4210107660"],"apc_list":null,"apc_paid":null,"fwci":1.9227,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.90093763,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"365","last_page":"372"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14254","display_name":"Digital Media and Visual Art","score":0.9117000102996826,"subfield":{"id":"https://openalex.org/subfields/1704","display_name":"Computer Graphics and Computer-Aided Design"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14254","display_name":"Digital Media and Visual Art","score":0.9117000102996826,"subfield":{"id":"https://openalex.org/subfields/1704","display_name":"Computer Graphics and Computer-Aided Design"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.909632682800293},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.7303481101989746},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5966445207595825},{"id":"https://openalex.org/keywords/quality-assessment","display_name":"Quality assessment","score":0.5864987969398499},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.5739514827728271},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5569712519645691},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.4873083829879761},{"id":"https://openalex.org/keywords/image-quality","display_name":"Image quality","score":0.4848874509334564},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.4551321566104889},{"id":"https://openalex.org/keywords/mean-opinion-score","display_name":"Mean opinion score","score":0.4432508945465088},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.42216771841049194},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.325760155916214},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.2963522970676422},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2635608911514282},{"id":"https://openalex.org/keywords/evaluation-methods","display_name":"Evaluation methods","score":0.18347039818763733},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1306551992893219},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.0738479495048523}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.909632682800293},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.7303481101989746},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5966445207595825},{"id":"https://openalex.org/C3020001037","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assessment","level":3,"score":0.5864987969398499},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.5739514827728271},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5569712519645691},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.4873083829879761},{"id":"https://openalex.org/C55020928","wikidata":"https://www.wikidata.org/wiki/Q3813865","display_name":"Image quality","level":3,"score":0.4848874509334564},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.4551321566104889},{"id":"https://openalex.org/C62897895","wikidata":"https://www.wikidata.org/wiki/Q1915482","display_name":"Mean opinion score","level":3,"score":0.4432508945465088},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42216771841049194},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.325760155916214},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2963522970676422},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2635608911514282},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.18347039818763733},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1306551992893219},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0738479495048523},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5220/0012462000003660","is_oa":true,"landing_page_url":"https://doi.org/10.5220/0012462000003660","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 19th International Joint Conference on Computer Vision, Imaging and Computer Graphics Theory and Applications","raw_type":"proceedings-article"},{"id":"pmh:oai:boa.unimib.it:10281/469898","is_oa":true,"landing_page_url":"https://www.scitepress.org/ProceedingsDetails.aspx?ID=MHRKKMhq/s8=&t=1","pdf_url":null,"source":{"id":"https://openalex.org/S4306401259","display_name":"BOA (University of Milano-Bicocca)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I66752286","host_organization_name":"University of Milano-Bicocca","host_organization_lineage":["https://openalex.org/I66752286"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.5220/0012462000003660","is_oa":true,"landing_page_url":"https://doi.org/10.5220/0012462000003660","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 19th International Joint Conference on Computer Vision, Imaging and Computer Graphics Theory and Applications","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2466958844","https://openalex.org/W2104070163","https://openalex.org/W2804751933","https://openalex.org/W2245293081","https://openalex.org/W1968377804","https://openalex.org/W1973829424","https://openalex.org/W4226186675","https://openalex.org/W3209787365","https://openalex.org/W1963681152","https://openalex.org/W4287019296"],"abstract_inverted_index":{"Image":[0],"quality":[1],"assessment":[2],"(IQA)":[3],"is":[4,37,94,113,160],"widely":[5],"used":[6],"to":[7,76,114],"evaluate":[8],"the":[9,19,95,104,110,137,158,190,194,203,208,213],"results":[10,201],"of":[11,21,90,97,106,109,207],"image":[12,67,98],"processing":[13,68],"methods.":[14],"While":[15],"in":[16,84,103,168,215],"recent":[17],"years":[18],"development":[20],"objective":[22],"IQA":[23,36,43,78,147,188,211],"metrics":[24],"has":[25,44],"seen":[26],"much":[27],"progress,":[28],"there":[29,70],"are":[30,71,135,148,176,181,197],"still":[31,72],"many":[32],"tasks":[33],"where":[34],"subjective":[35,42,77,146,187,210],"significantly":[38],"more":[39,47],"preferred.":[40],"Using":[41],"become":[45,61],"even":[46],"attractive":[48],"ever":[49],"since":[50],"crowdsourcing":[51],"platforms":[52],"such":[53,91,140],"as":[54,118],"Amazon":[55],"Mechanical":[56],"Turk":[57],"and":[58,150,174,178,193,205],"Toloka":[59,191],"have":[60,80,127],"available.":[62],"However,":[63],"for":[64,139,212],"some":[65,73],"specific":[66],"tasks,":[69],"questions":[74,125],"related":[75],"that":[79,126],"not":[81,128],"been":[82,129],"solved":[83],"a":[85,92,119,141],"satisfactory":[86],"way.":[87],"An":[88],"example":[89],"task":[93,142],"evaluation":[96,159],"rendering":[99],"styles":[100,112,175],"where,":[101],"unlike":[102],"case":[105],"distortions,":[107],"none":[108],"evaluated":[111,183],"be":[115],"objectively":[116],"regarded":[117],"priori":[120],"better":[121],"or":[122],"worse.":[123],"The":[124],"properly":[130],"answered":[131],"up":[132],"until":[133],"now":[134],"whether":[136,151],"scores":[138,196],"obtained":[143,195],"through":[144],"crowdsourced":[145,186,209],"reliable":[149],"they":[152,180],"remain":[153],"stable,":[154],"i.e.,":[155],"similar":[156],"if":[157],"repeated":[161],"over":[162],"time.":[163],"To":[164],"answer":[165],"these":[166],"questions,":[167],"this":[169],"paper":[170],"first":[171],"several":[172],"images":[173],"selected":[177],"defined,":[179],"then":[182],"by":[184],"using":[185],"on":[189],"platform,":[192],"numerically":[198],"analyzed.":[199],"Experimental":[200],"confirm":[202],"reliability":[204],"stability":[206],"problem":[214],"question.":[216]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2025-12-21T01:58:51.020947","created_date":"2025-10-10T00:00:00"}