{"id":"https://openalex.org/W3154677174","doi":"https://doi.org/10.1109/iccv48922.2021.00937","title":"Solving Inefficiency of Self-supervised Representation Learning","display_name":"Solving Inefficiency of Self-supervised Representation Learning","publication_year":2021,"publication_date":"2021-10-01","ids":{"openalex":"https://openalex.org/W3154677174","doi":"https://doi.org/10.1109/iccv48922.2021.00937","mag":"3154677174"},"language":"en","primary_location":{"id":"doi:10.1109/iccv48922.2021.00937","doi":"https://doi.org/10.1109/iccv48922.2021.00937","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv48922.2021.00937","pdf_url":null,"source":{"id":"https://openalex.org/S4363607764","display_name":"IEEE/CVF International Conference on Computer Vision (ICCV)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2104.08760","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052611320","display_name":"Guangrun Wang","orcid":"https://orcid.org/0000-0001-7760-1339","observed_orcids":["https://orcid.org/0000-0001-7760-1339"]},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]},{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]},{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["CN","GB","SG"],"is_corresponding":false,"raw_author_name":"Guangrun Wang","raw_affiliation_strings":["Nanyang Technological University","Sun Yat-sen University","University of Oxford"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanyang Technological University","institution_ids":["https://openalex.org/I172675005"]},{"raw_affiliation_string":"Sun Yat-sen University","institution_ids":["https://openalex.org/I157773358"]},{"raw_affiliation_string":"University of Oxford","institution_ids":["https://openalex.org/I40120149"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088124671","display_name":"Keze Wang","orcid":"https://orcid.org/0000-0002-7817-8306","observed_orcids":["https://orcid.org/0000-0002-7817-8306"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Keze Wang","raw_affiliation_strings":["DarkMatter AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"DarkMatter AI Research","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028380963","display_name":"Guangcong Wang","orcid":"https://orcid.org/0000-0002-6627-814X","observed_orcids":["https://orcid.org/0000-0002-6627-814X"]},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]},{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["CN","SG"],"is_corresponding":false,"raw_author_name":"Guangcong Wang","raw_affiliation_strings":["Nanyang Technological University","Sun Yat-sen University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nanyang Technological University","institution_ids":["https://openalex.org/I172675005"]},{"raw_affiliation_string":"Sun Yat-sen University","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001534492","display_name":"Philip H. S. Torr","orcid":"https://orcid.org/0009-0006-0259-5732","observed_orcids":["https://orcid.org/0009-0006-0259-5732"]},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Philip H.S. Torr","raw_affiliation_strings":["University of Oxford"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oxford","institution_ids":["https://openalex.org/I40120149"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100412937","display_name":"Liang Lin","orcid":"https://orcid.org/0000-0003-2248-3755","observed_orcids":["https://orcid.org/0000-0003-2248-3755","https://orcid.org/0000-0001-5704-4168"]},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liang Lin","raw_affiliation_strings":["Sun Yat-sen University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sun Yat-sen University","institution_ids":["https://openalex.org/I157773358"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.0769,"has_fulltext":true,"cited_by_count":68,"citation_normalized_percentile":{"value":0.96453585,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":92,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"9485","last_page":"9495"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9678000211715698,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9678000211715698,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.017999999225139618,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.003599999938160181,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/self-supervised-representation-learning","display_name":"self-supervised representation learning","score":0.9390000104904175},{"id":"https://openalex.org/keywords/contrastive-learning","display_name":"contrastive learning","score":1.0},{"id":"https://openalex.org/keywords/learning-efficiency","display_name":"learning efficiency","score":0.8149999976158142},{"id":"https://openalex.org/keywords/negative-sample-selection","display_name":"negative sample selection","score":0.9330000281333923}],"concepts":[{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.7348121404647827},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6873636245727539},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6488256454467773},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.6023097038269043},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5594741106033325},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.4762271046638489},{"id":"https://openalex.org/C8038995","wikidata":"https://www.wikidata.org/wiki/Q1152135","display_name":"Unsupervised learning","level":2,"score":0.46963831782341003},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.46136751770973206},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.42966681718826294},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.4116096794605255},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/iccv48922.2021.00937","doi":"https://doi.org/10.1109/iccv48922.2021.00937","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iccv48922.2021.00937","pdf_url":null,"source":{"id":"https://openalex.org/S4363607764","display_name":"IEEE/CVF International Conference on Computer Vision (ICCV)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE/CVF International Conference on Computer Vision (ICCV)","raw_type":"proceedings-article"},{"id":"pmh:oai:ora.ox.ac.uk:uuid:a795d26c-1d4f-40d6-aa9a-0321dca70f0f","doi":null,"is_oa":false,"landing_page_url":"https://ora.ox.ac.uk/objects/uuid:a795d26c-1d4f-40d6-aa9a-0321dca70f0f","pdf_url":null,"source":{"id":"https://openalex.org/S4306402636","display_name":"Oxford University Research Archive (ORA) (University of Oxford)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I40120149","host_organization_name":"University of Oxford","host_organization_lineage":["https://openalex.org/I40120149"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"Symplectic Elements","raw_type":"http://purl.org/coar/resource_type/c_5794"},{"id":"pmh:oai:arXiv.org:2104.08760","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2104.08760","pdf_url":"https://arxiv.org/pdf/2104.08760","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2104.08760","doi":"https://doi.org/10.48550/arxiv.2104.08760","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2104.08760","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2104.08760","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2104.08760","pdf_url":"https://arxiv.org/pdf/2104.08760","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1295000622","display_name":null,"funder_award_id":"61836012","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1409295806","display_name":null,"funder_award_id":"2017A030312006","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2141641167","display_name":null,"funder_award_id":"U1811463","funder_id":"https://openalex.org/F4320321921","funder_display_name":"Natural Science Foundation of Guangdong Province"},{"id":"https://openalex.org/G3146272683","display_name":null,"funder_award_id":"#U1811463","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6473094083","display_name":null,"funder_award_id":"2020AAA0109700","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8471144913","display_name":null,"funder_award_id":"2019A1515010939","funder_id":"https://openalex.org/F4320321921","funder_display_name":"Natural Science Foundation of Guangdong Province"},{"id":"https://openalex.org/G8558289817","display_name":null,"funder_award_id":"61876224","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G9000061443","display_name":null,"funder_award_id":"2017A030312006","funder_id":"https://openalex.org/F4320321921","funder_display_name":"Natural Science Foundation of Guangdong Province"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321921","display_name":"Natural Science Foundation of Guangdong Province","ror":null},{"id":"https://openalex.org/F4320335087","display_name":"Innovate UK","ror":"https://ror.org/05ar5fy68"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3154677174.pdf","grobid_xml":"https://content.openalex.org/works/W3154677174.grobid-xml"},"referenced_works_count":58,"referenced_works":["https://openalex.org/W343636949","https://openalex.org/W639708223","https://openalex.org/W1861492603","https://openalex.org/W1971955426","https://openalex.org/W1979260620","https://openalex.org/W2025768430","https://openalex.org/W2089074647","https://openalex.org/W2096733369","https://openalex.org/W2117539524","https://openalex.org/W2138621090","https://openalex.org/W2144796873","https://openalex.org/W2148349024","https://openalex.org/W2155893237","https://openalex.org/W2321533354","https://openalex.org/W2326925005","https://openalex.org/W2342877626","https://openalex.org/W2592691248","https://openalex.org/W2598634450","https://openalex.org/W2599723140","https://openalex.org/W2785325870","https://openalex.org/W2785717964","https://openalex.org/W2795758732","https://openalex.org/W2883725317","https://openalex.org/W2896234850","https://openalex.org/W2950220847","https://openalex.org/W2951292523","https://openalex.org/W2962830209","https://openalex.org/W2962996864","https://openalex.org/W2963074118","https://openalex.org/W2963150697","https://openalex.org/W2963975998","https://openalex.org/W2964201641","https://openalex.org/W2970971581","https://openalex.org/W2986405467","https://openalex.org/W2994536315","https://openalex.org/W2995181141","https://openalex.org/W3003477032","https://openalex.org/W3005680577","https://openalex.org/W3008503415","https://openalex.org/W3009561768","https://openalex.org/W3026092005","https://openalex.org/W3034372982","https://openalex.org/W3034576826","https://openalex.org/W3035014997","https://openalex.org/W3035060554","https://openalex.org/W3035524453","https://openalex.org/W3036224891","https://openalex.org/W3039465721","https://openalex.org/W3043746735","https://openalex.org/W3080922457","https://openalex.org/W3091940082","https://openalex.org/W3093274308","https://openalex.org/W3106743555","https://openalex.org/W3112466543","https://openalex.org/W3123939835","https://openalex.org/W3171007011","https://openalex.org/W4239510810","https://openalex.org/W4297808394"],"related_works":["https://openalex.org/W3177090610","https://openalex.org/W4381611929","https://openalex.org/W7052485320","https://openalex.org/W4361251611","https://openalex.org/W3005680577","https://openalex.org/W3106743555","https://openalex.org/W7164896846","https://openalex.org/W4410536657","https://openalex.org/W4391096036","https://openalex.org/W4404446081"],"abstract_inverted_index":{"Self-supervised":[0],"learning":[1,15,28,34,46,61,74,98,147,227],"(especially":[2],"contrastive":[3,27,60,97],"learning)":[4],"has":[5],"attracted":[6],"great":[7],"interest":[8],"due":[9],"to":[10,73,84,101,125,161,173,194],"its":[11],"huge":[12],"potential":[13],"in":[14,18,59,210],"discriminative":[16],"representations":[17],"an":[19],"unsupervised":[20],"manner.":[21],"Despite":[22],"the":[23,79,86,92,104,111,123,129,163,167,175,181,196,201,226,230],"acknowledged":[24],"successes,":[25],"existing":[26],"methods":[29,233],"suffer":[30],"from":[31,117,190],"very":[32],"low":[33],"efficiency,":[35],"e.g.,":[36],"taking":[37],"about":[38],"ten":[39],"times":[40],"more":[41],"training":[42],"epochs":[43],"than":[44],"supervised":[45],"for":[47,96],"comparable":[48],"recognition":[49],"accuracy.":[50],"In":[51],"this":[52],"paper,":[53],"we":[54,63,142,155,179],"reveal":[55],"two":[56,140],"contradictory":[57],"phenomena":[58],"that":[62,78,110],"call":[64],"under-clustering":[65,176],"and":[66,170,178,217],"over-clustering":[67,197],"problems,":[68,141],"which":[69],"are":[70,99],"major":[71],"obstacles":[72],"efficiency.":[75],"Under-clustering":[76],"means":[77],"model":[80,112,124],"cannot":[81,113],"efficiently":[82,114],"learn":[83,115],"discover":[85],"dissimilarity":[87],"between":[88,166],"inter-class":[89],"samples":[90,127,193],"when":[91],"negative":[93,119,171,182,187,192],"sample":[94,120,188],"pairs":[95,172],"insufficient":[100],"differentiate":[102],"all":[103,191],"actual":[105,131],"object":[106],"classes.":[107],"Over-clustering":[108],"implies":[109],"features":[116],"excessive":[118],"pairs,":[121],"forcing":[122],"over-cluster":[126],"of":[128],"same":[130],"classes":[132],"into":[133],"different":[134],"clusters.":[135],"To":[136],"simultaneously":[137],"overcome":[138],"these":[139],"propose":[143],"a":[144,150,157,186,235],"novel":[145],"self-supervised":[146],"framework":[148,209],"using":[149],"truncated":[151],"triplet":[152,158],"loss.":[153],"Precisely,":[154],"employ":[156],"loss":[159],"tending":[160],"maximize":[162],"relative":[164],"distance":[165],"positive":[168],"pair":[169,183],"address":[174],"problem;":[177],"construct":[180],"by":[184,200,234],"selecting":[185],"deputy":[189],"avoid":[195],"problem,":[198],"guaranteed":[199],"Bernoulli":[202],"Distribution":[203],"model.":[204],"We":[205],"extensively":[206],"evaluate":[207],"our":[208,222],"several":[211],"large-scale":[212],"benchmarks":[213],"(e.g.,":[214,225],"ImageNet,":[215],"SYSU-30k,":[216],"COCO).":[218],"The":[219],"results":[220],"demonstrate":[221],"model\u2019s":[223],"superiority":[224],"efficiency)":[228],"over":[229],"latest":[231],"state-of-the-art":[232],"clear":[236],"margin.":[237],"See":[238],"Codes1.":[239]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":25},{"year":2022,"cited_by_count":13},{"year":2021,"cited_by_count":2}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2021-04-26T00:00:00"}