10 · Embeddings
References
Distributional Structure — Zellig Harris. Word 10(2–3), 1954.
@article{harris1954distributional,
title = {Distributional Structure},
author = {Harris, Zellig S.},
journal = {Word},
volume = {10},
number = {2-3},
pages = {146--162},
year = {1954}
}A Neural Probabilistic Language Model — Bengio, Ducharme, Vincent & Janvin. JMLR 3, 2003.
@article{bengio2003neural,
title = {A Neural Probabilistic Language Model},
author = {Bengio, Yoshua and Ducharme, R{\'e}jean and Vincent, Pascal and Janvin, Christian},
journal = {Journal of Machine Learning Research},
volume = {3},
pages = {1137--1155},
year = {2003}
}Efficient Estimation of Word Representations in Vector Space — Mikolov, Chen, Corrado & Dean. 2013.
@article{mikolov2013efficient,
title = {Efficient Estimation of Word Representations in Vector Space},
author = {Mikolov, Tomas and Chen, Kai and Corrado, Greg and Dean, Jeffrey},
journal = {arXiv preprint arXiv:1301.3781},
year = {2013},
eprint = {1301.3781},
archivePrefix = {arXiv}
}Distributed Representations of Words and Phrases and their Compositionality — Mikolov, Sutskever, Chen, Corrado & Dean. NeurIPS 2013.
@inproceedings{mikolov2013distributed,
title = {Distributed Representations of Words and Phrases and their Compositionality},
author = {Mikolov, Tomas and Sutskever, Ilya and Chen, Kai and Corrado, Greg and Dean, Jeffrey},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
year = {2013},
eprint = {1310.4546},
archivePrefix = {arXiv}
}Linguistic Regularities in Continuous Space Word Representations — Mikolov, Yih & Zweig. NAACL 2013.
@inproceedings{mikolov2013linguistic,
title = {Linguistic Regularities in Continuous Space Word Representations},
author = {Mikolov, Tomas and Yih, Wen-tau and Zweig, Geoffrey},
booktitle = {North American Chapter of the Association for Computational Linguistics (NAACL)},
pages = {746--751},
year = {2013}
}GloVe: Global Vectors for Word Representation — Pennington, Socher & Manning. EMNLP 2014.
@inproceedings{pennington2014glove,
title = {GloVe: Global Vectors for Word Representation},
author = {Pennington, Jeffrey and Socher, Richard and Manning, Christopher D.},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
pages = {1532--1543},
year = {2014}
}Neural Word Embedding as Implicit Matrix Factorization — Levy & Goldberg. NeurIPS 2014.
@inproceedings{levy2014neural,
title = {Neural Word Embedding as Implicit Matrix Factorization},
author = {Levy, Omer and Goldberg, Yoav},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
year = {2014}
}Improving Distributional Similarity with Lessons Learned from Word Embeddings — Levy, Goldberg & Dagan. TACL 3, 2015.
@article{levy2015improving,
title = {Improving Distributional Similarity with Lessons Learned from Word Embeddings},
author = {Levy, Omer and Goldberg, Yoav and Dagan, Ido},
journal = {Transactions of the Association for Computational Linguistics},
volume = {3},
pages = {211--225},
year = {2015}
}Enriching Word Vectors with Subword Information — Bojanowski, Grave, Joulin & Mikolov. TACL 5, 2017.
@article{bojanowski2017enriching,
title = {Enriching Word Vectors with Subword Information},
author = {Bojanowski, Piotr and Grave, Edouard and Joulin, Armand and Mikolov, Tomas},
journal = {Transactions of the Association for Computational Linguistics},
volume = {5},
pages = {135--146},
year = {2017},
eprint = {1607.04606},
archivePrefix = {arXiv}
}Neural Machine Translation of Rare Words with Subword Units — Sennrich, Haddow & Birch. ACL 2016.
@inproceedings{sennrich2016neural,
title = {Neural Machine Translation of Rare Words with Subword Units},
author = {Sennrich, Rico and Haddow, Barry and Birch, Alexandra},
booktitle = {Annual Meeting of the Association for Computational Linguistics (ACL)},
year = {2016},
eprint = {1508.07909},
archivePrefix = {arXiv}
}Deep Contextualized Word Representations — Peters et al. NAACL 2018.
@inproceedings{peters2018deep,
title = {Deep Contextualized Word Representations},
author = {Peters, Matthew E. and Neumann, Mark and Iyyer, Mohit and Gardner, Matt
and Clark, Christopher and Lee, Kenton and Zettlemoyer, Luke},
booktitle = {North American Chapter of the Association for Computational Linguistics (NAACL)},
year = {2018},
eprint = {1802.05365},
archivePrefix = {arXiv}
}BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding — Devlin, Chang, Lee & Toutanova. NAACL 2019.
@inproceedings{devlin2019bert,
title = {BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding},
author = {Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
booktitle = {North American Chapter of the Association for Computational Linguistics (NAACL)},
year = {2019},
eprint = {1810.04805},
archivePrefix = {arXiv}
}Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks — Reimers & Gurevych. EMNLP 2019.
@inproceedings{reimers2019sentence,
title = {Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks},
author = {Reimers, Nils and Gurevych, Iryna},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
year = {2019},
eprint = {1908.10084},
archivePrefix = {arXiv}
}Man is to Computer Programmer as Woman is to Homemaker? Debiasing Word Embeddings — Bolukbasi, Chang, Zou, Saligrama & Kalai. NeurIPS 2016.
@inproceedings{bolukbasi2016man,
title = {Man is to Computer Programmer as Woman is to Homemaker? Debiasing Word Embeddings},
author = {Bolukbasi, Tolga and Chang, Kai-Wei and Zou, James and Saligrama, Venkatesh and Kalai, Adam},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
year = {2016},
eprint = {1607.06520},
archivePrefix = {arXiv}
}Japanese and Korean Voice Search — Schuster & Nakajima. ICASSP 2012. The origin of WordPiece.
@inproceedings{schuster2012japanese,
title = {Japanese and Korean Voice Search},
author = {Schuster, Mike and Nakajima, Kaisuke},
booktitle = {IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
pages = {5149--5152},
year = {2012}
}Linguistic Regularities in Sparse and Explicit Word Representations — Levy & Goldberg. CoNLL 2014. Introduces 3CosMul.
@inproceedings{levy2014linguistic,
title = {Linguistic Regularities in Sparse and Explicit Word Representations},
author = {Levy, Omer and Goldberg, Yoav},
booktitle = {Conference on Computational Natural Language Learning (CoNLL)},
pages = {171--180},
year = {2014}
}Measuring Word Significance using Distributed Representations of Words — Schakel & Wilson. 2015.
@article{schakel2015measuring,
title = {Measuring Word Significance using Distributed Representations of Words},
author = {Schakel, Adriaan M. J. and Wilson, Benjamin J.},
journal = {arXiv preprint arXiv:1508.02297},
year = {2015},
eprint = {1508.02297},
archivePrefix = {arXiv}
}Issues in Evaluating Semantic Spaces using Word Analogies — Linzen. RepEval 2016.
@inproceedings{linzen2016issues,
title = {Issues in Evaluating Semantic Spaces using Word Analogies},
author = {Linzen, Tal},
booktitle = {Proceedings of the 1st Workshop on Evaluating Vector-Space Representations for NLP (RepEval)},
pages = {13--18},
year = {2016}
}Fair Is Better than Sensational: Man Is to Doctor as Woman Is to Doctor — Nissim, van Noord & van der Goot. Computational Linguistics 46(2), 2020.
@article{nissim2020fair,
title = {Fair Is Better than Sensational: Man Is to Doctor as Woman Is to Doctor},
author = {Nissim, Malvina and van Noord, Rik and van der Goot, Rob},
journal = {Computational Linguistics},
volume = {46},
number = {2},
pages = {487--497},
year = {2020}
}Linear Algebraic Structure of Word Senses, with Applications to Polysemy — Arora, Li, Liang, Ma & Risteski. TACL 6, 2018.
@article{arora2018linear,
title = {Linear Algebraic Structure of Word Senses, with Applications to Polysemy},
author = {Arora, Sanjeev and Li, Yuanzhi and Liang, Yingyu and Ma, Tengyu and Risteski, Andrej},
journal = {Transactions of the Association for Computational Linguistics},
volume = {6},
year = {2018},
eprint = {1601.03764},
archivePrefix = {arXiv}
}Semantics Derived Automatically from Language Corpora Contain Human-like Biases — Caliskan, Bryson & Narayanan. Science 356(6334), 2017.
@article{caliskan2017semantics,
title = {Semantics Derived Automatically from Language Corpora Contain Human-like Biases},
author = {Caliskan, Aylin and Bryson, Joanna J. and Narayanan, Arvind},
journal = {Science},
volume = {356},
number = {6334},
pages = {183--186},
year = {2017},
eprint = {1608.07187},
archivePrefix = {arXiv}
}How Contextual are Contextualized Word Representations? Comparing the Geometry of BERT, ELMo, and GPT-2 Embeddings — Ethayarajh. EMNLP 2019.
@inproceedings{ethayarajh2019contextual,
title = {How Contextual are Contextualized Word Representations? Comparing the Geometry of BERT, ELMo, and GPT-2 Embeddings},
author = {Ethayarajh, Kawin},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
year = {2019},
eprint = {1909.00512},
archivePrefix = {arXiv}
}Dense Passage Retrieval for Open-Domain Question Answering — Karpukhin et al. EMNLP 2020.
@inproceedings{karpukhin2020dense,
title = {Dense Passage Retrieval for Open-Domain Question Answering},
author = {Karpukhin, Vladimir and O{\u{g}}uz, Barlas and Min, Sewon and Lewis, Patrick and Wu, Ledell
and Edunov, Sergey and Chen, Danqi and Yih, Wen-tau},
booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
year = {2020},
eprint = {2004.04906},
archivePrefix = {arXiv}
}Matryoshka Representation Learning — Kusupati et al. NeurIPS 2022.
@inproceedings{kusupati2022matryoshka,
title = {Matryoshka Representation Learning},
author = {Kusupati, Aditya and Bhatt, Gantavya and Rege, Aniket and Wallingford, Matthew and Sinha, Aditya
and Ramanujan, Vivek and Howard-Snyder, William and Chen, Kaifeng and Kakade, Sham
and Jain, Prateek and Farhadi, Ali},
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
year = {2022},
eprint = {2205.13147},
archivePrefix = {arXiv}
}Text Embeddings by Weakly-Supervised Contrastive Pre-training — Wang et al. 2022. The E5 models.
@article{wang2022text,
title = {Text Embeddings by Weakly-Supervised Contrastive Pre-training},
author = {Wang, Liang and Yang, Nan and Huang, Xiaolong and Jiao, Binxing and Yang, Linjun
and Jiang, Daxin and Majumder, Rangan and Wei, Furu},
journal = {arXiv preprint arXiv:2212.03533},
year = {2022},
eprint = {2212.03533},
archivePrefix = {arXiv}
}MTEB: Massive Text Embedding Benchmark — Muennighoff, Tazi, Magne & Reimers. EACL 2023.
@inproceedings{muennighoff2023mteb,
title = {MTEB: Massive Text Embedding Benchmark},
author = {Muennighoff, Niklas and Tazi, Nouamane and Magne, Lo{\"i}c and Reimers, Nils},
booktitle = {European Chapter of the Association for Computational Linguistics (EACL)},
year = {2023},
eprint = {2210.07316},
archivePrefix = {arXiv}
}Speech and Language Processing, chapter 6: Vector Semantics and Embeddings — Jurafsky & Martin. 3rd edition draft.
@book{jurafsky2024speech,
title = {Speech and Language Processing},
author = {Jurafsky, Daniel and Martin, James H.},
edition = {3rd edition draft},
year = {2024},
url = {https://web.stanford.edu/~jurafsky/slp3/}
}The Illustrated Word2vec — Jay Alammar. 2019.