{"id":"https://openalex.org/W2345254477","doi":"https://doi.org/10.1109/tmm.2016.2559942","title":"A Reliable Methodology to Collect Ground Truth Data of Image Aesthetic Appeal","display_name":"A Reliable Methodology to Collect Ground Truth Data of Image Aesthetic Appeal","publication_year":2016,"publication_date":"2016-04-28","ids":{"openalex":"https://openalex.org/W2345254477","doi":"https://doi.org/10.1109/tmm.2016.2559942","mag":"2345254477"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2016.2559942","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2016.2559942","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039442748","display_name":"Ernestasia Siahaan","orcid":"https://orcid.org/0000-0003-0552-9483"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Ernestasia Siahaan","raw_affiliation_strings":["Multimedia Computing Group, Intelligent Systems Department, Delft University of Technology, Delft, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Multimedia Computing Group, Intelligent Systems Department, Delft University of Technology, Delft, The Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037486611","display_name":"Alan Hanjali\u0107","orcid":"https://orcid.org/0000-0002-5771-2549"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Alan Hanjalic","raw_affiliation_strings":["Multimedia Computing Group, Intelligent Systems Department, Delft University of Technology, Delft, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Multimedia Computing Group, Intelligent Systems Department, Delft University of Technology, Delft, The Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019183502","display_name":"Judith Redi","orcid":"https://orcid.org/0000-0003-1936-8860"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Judith Redi","raw_affiliation_strings":["Multimedia Computing Group, Intelligent Systems Department, Delft University of Technology, Delft, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Multimedia Computing Group, Intelligent Systems Department, Delft University of Technology, Delft, The Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5039442748"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":2.7032,"has_fulltext":false,"cited_by_count":34,"citation_normalized_percentile":{"value":0.93498937,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"18","issue":"7","first_page":"1338","last_page":"1350"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11605","display_name":"Visual Attention and Saliency Detection","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12650","display_name":"Aesthetic Perception and Analysis","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/appeal","display_name":"Appeal","score":0.8879550695419312},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8644601106643677},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.835200846195221},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6958200931549072},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.5012426376342773},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4660654366016388},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.4500194787979126},{"id":"https://openalex.org/keywords/common-ground","display_name":"Common ground","score":0.43333712220191956},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.4200168251991272},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.40720611810684204},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.39215314388275146},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.32219183444976807},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1608368158340454},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.14965060353279114},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1475549042224884},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.13660871982574463},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.13218513131141663}],"concepts":[{"id":"https://openalex.org/C2778449503","wikidata":"https://www.wikidata.org/wiki/Q329777","display_name":"Appeal","level":2,"score":0.8879550695419312},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8644601106643677},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.835200846195221},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6958200931549072},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.5012426376342773},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4660654366016388},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.4500194787979126},{"id":"https://openalex.org/C2777877512","wikidata":"https://www.wikidata.org/wiki/Q1116097","display_name":"Common ground","level":2,"score":0.43333712220191956},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.4200168251991272},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40720611810684204},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.39215314388275146},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.32219183444976807},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1608368158340454},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.14965060353279114},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1475549042224884},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.13660871982574463},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.13218513131141663},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2016.2559942","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2016.2559942","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"No poverty","id":"https://metadata.un.org/sdg/1","score":0.550000011920929}],"awards":[{"id":"https://openalex.org/G8179775404","display_name":null,"funder_award_id":"639.021.230","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W91524499","https://openalex.org/W1492162446","https://openalex.org/W1510555363","https://openalex.org/W1511924373","https://openalex.org/W1595637775","https://openalex.org/W1597066552","https://openalex.org/W1966700952","https://openalex.org/W1977476420","https://openalex.org/W1985364666","https://openalex.org/W1986525094","https://openalex.org/W1987931295","https://openalex.org/W1988119253","https://openalex.org/W1996776980","https://openalex.org/W2013861934","https://openalex.org/W2019607802","https://openalex.org/W2020847903","https://openalex.org/W2024415937","https://openalex.org/W2026874689","https://openalex.org/W2033442452","https://openalex.org/W2040141351","https://openalex.org/W2044885272","https://openalex.org/W2053442087","https://openalex.org/W2056380823","https://openalex.org/W2057137426","https://openalex.org/W2063948594","https://openalex.org/W2067197793","https://openalex.org/W2078807908","https://openalex.org/W2081059637","https://openalex.org/W2089242886","https://openalex.org/W2101970752","https://openalex.org/W2111993661","https://openalex.org/W2124542849","https://openalex.org/W2128378281","https://openalex.org/W2129375743","https://openalex.org/W2136585056","https://openalex.org/W2142677209","https://openalex.org/W2146121144","https://openalex.org/W2153102006","https://openalex.org/W2153346601","https://openalex.org/W2155243985","https://openalex.org/W2163031058","https://openalex.org/W2170375455","https://openalex.org/W2171269138","https://openalex.org/W2566053671","https://openalex.org/W2996879375","https://openalex.org/W4231606319","https://openalex.org/W4252283762","https://openalex.org/W6630520941","https://openalex.org/W6653998682","https://openalex.org/W6685125350"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W1490650877","https://openalex.org/W4307535225","https://openalex.org/W2905544777"],"abstract_inverted_index":{"Recognizing":[0],"what":[1],"makes":[2],"an":[3],"image":[4,22,131,178],"aesthetically":[5],"pleasing":[6],"is":[7],"crucial":[8],"to":[9,20,54,100,129,198],"the":[10,36,66,96,138,165,200],"effectiveness":[11],"of":[12,31,52,104,142,202,205,212],"many":[13],"multimedia":[14],"systems.":[15],"Several":[16],"works":[17],"have":[18],"attempted":[19],"build":[21],"aesthetic":[23,88,105,132,143,179,213],"appeal":[24,89,106,133,144,180,214],"predictors,":[25],"and":[26,125,140,151,191],"created":[27],"their":[28,74],"own":[29],"set":[30],"ground":[32,102,134,206],"truth":[33,103,135,207],"data":[34,107,136,208],"for":[35,171,209],"purpose,":[37],"either":[38],"by":[39,48],"using":[40],"rated":[41],"images":[42,56],"from":[43],"photo":[44],"sharing":[45],"websites,":[46],"or":[47,59],"asking":[49],"a":[50,115],"pool":[51],"users":[53],"rate":[55],"in":[57,81],"lab":[58,190],"crowdsourcing":[60],"experiments.":[61],"Literature":[62],"has":[63],"shown":[64],"that":[65,118,149,177],"way":[67],"these":[68],"experiments":[69],"are":[70,108],"conducted":[71],"can":[72,79],"influence":[73,137],"results:":[75],"poor":[76],"experimental":[77,123,187],"setup":[78],"result":[80],"poorly":[82],"reliable":[83,162,170],"outcomes":[84],"(i.e.,":[85,189],"highly":[86],"imprecise":[87],"measures).":[90],"A":[91],"question":[92],"then":[93],"arises":[94],"whether":[95],"different":[97,122,186],"choices":[98],"made":[99],"collect":[101,130],"appropriate.":[109],"In":[110],"this":[111],"paper,":[112],"we":[113],"propose":[114],"systematic":[116],"study":[117],"looks":[119],"into":[120],"how":[121],"environments":[124,188],"rating":[126,158],"scales":[127,153],"used":[128],"reliability":[139],"repeatability":[141],"assessments.":[145],"Our":[146],"findings":[147],"show":[148,176],"discrete":[150],"continuous":[152,166],"with":[154,164],"five-point":[155],"absolute":[156],"category":[157],"labels":[159],"yield":[160],"more":[161,169],"results,":[163],"scale":[167],"being":[168],"abstract":[172],"images.":[173],"We":[174,193],"also":[175],"assessments":[181],"could":[182],"be":[183],"repeatable":[184],"across":[185],"crowdsourcing).":[192],"finally":[194],"formulate":[195],"concrete":[196],"recommendations":[197],"guide":[199],"collection":[201],"large":[203],"sets":[204],"training":[210],"models":[211],"appreciation.":[215]},"counts_by_year":[{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":8},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":10}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}