References

Neural Machine Translation by Jointly Learning to Align and Translate — Bahdanau, Cho & Bengio. ICLR 2015.

@inproceedings{bahdanau2015neural,
  title     = {Neural Machine Translation by Jointly Learning to Align and Translate},
  author    = {Bahdanau, Dzmitry and Cho, Kyunghyun and Bengio, Yoshua},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year      = {2015},
  eprint    = {1409.0473},
  archivePrefix = {arXiv}
}

Sequence to Sequence Learning with Neural Networks — Sutskever, Vinyals & Le. NeurIPS 2014.

@inproceedings{sutskever2014sequence,
  title     = {Sequence to Sequence Learning with Neural Networks},
  author    = {Sutskever, Ilya and Vinyals, Oriol and Le, Quoc V.},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year      = {2014},
  eprint    = {1409.3215},
  archivePrefix = {arXiv}
}

Effective Approaches to Attention-based Neural Machine Translation — Luong, Pham & Manning. EMNLP 2015.

@inproceedings{luong2015effective,
  title     = {Effective Approaches to Attention-based Neural Machine Translation},
  author    = {Luong, Minh-Thang and Pham, Hieu and Manning, Christopher D.},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  year      = {2015},
  eprint    = {1508.04025},
  archivePrefix = {arXiv}
}

Attention Is All You Need — Vaswani et al. NeurIPS 2017.

@inproceedings{vaswani2017attention,
  title     = {Attention Is All You Need},
  author    = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob
               and Jones, Llion and Gomez, Aidan N. and Kaiser, {\L}ukasz and Polosukhin, Illia},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year      = {2017},
  eprint    = {1706.03762},
  archivePrefix = {arXiv}
}

Deep Residual Learning for Image Recognition — He, Zhang, Ren & Sun. CVPR 2016.

@inproceedings{he2016deep,
  title     = {Deep Residual Learning for Image Recognition},
  author    = {He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
  year      = {2016},
  eprint    = {1512.03385},
  archivePrefix = {arXiv}
}

Layer Normalization — Ba, Kiros & Hinton. 2016.

@article{ba2016layer,
  title   = {Layer Normalization},
  author  = {Ba, Jimmy Lei and Kiros, Jamie Ryan and Hinton, Geoffrey E.},
  journal = {arXiv preprint arXiv:1607.06450},
  year    = {2016},
  eprint  = {1607.06450},
  archivePrefix = {arXiv}
}

On Layer Normalization in the Transformer Architecture — Xiong et al. ICML 2020.

@inproceedings{xiong2020layer,
  title     = {On Layer Normalization in the Transformer Architecture},
  author    = {Xiong, Ruibin and Yang, Yunchang and He, Di and Zheng, Kai and Zheng, Shuxin
               and Xing, Chen and Zhang, Huishuai and Lan, Yanyan and Wang, Liwei and Liu, Tie-Yan},
  booktitle = {International Conference on Machine Learning (ICML)},
  year      = {2020},
  eprint    = {2002.04745},
  archivePrefix = {arXiv}
}

Self-Attention with Relative Position Representations — Shaw, Uszkoreit & Vaswani. NAACL 2018.

@inproceedings{shaw2018self,
  title     = {Self-Attention with Relative Position Representations},
  author    = {Shaw, Peter and Uszkoreit, Jakob and Vaswani, Ashish},
  booktitle = {North American Chapter of the Association for Computational Linguistics (NAACL)},
  year      = {2018},
  eprint    = {1803.02155},
  archivePrefix = {arXiv}
}

RoFormer: Enhanced Transformer with Rotary Position Embedding — Su et al. 2021.

@article{su2021roformer,
  title   = {RoFormer: Enhanced Transformer with Rotary Position Embedding},
  author  = {Su, Jianlin and Lu, Yu and Pan, Shengfeng and Murtadha, Ahmed and Wen, Bo and Liu, Yunfeng},
  journal = {arXiv preprint arXiv:2104.09864},
  year    = {2021},
  eprint  = {2104.09864},
  archivePrefix = {arXiv}
}

Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation — Press, Smith & Lewis. ICLR 2022.

@inproceedings{press2022train,
  title     = {Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation},
  author    = {Press, Ofir and Smith, Noah A. and Lewis, Mike},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year      = {2022},
  eprint    = {2108.12409},
  archivePrefix = {arXiv}
}

FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness — Dao, Fu, Ermon, Rudra & Ré. NeurIPS 2022.

@inproceedings{dao2022flashattention,
  title     = {FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness},
  author    = {Dao, Tri and Fu, Daniel Y. and Ermon, Stefano and Rudra, Atri and R{\'e}, Christopher},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year      = {2022},
  eprint    = {2205.14135},
  archivePrefix = {arXiv}
}

Longformer: The Long-Document Transformer — Beltagy, Peters & Cohan. 2020.

@article{beltagy2020longformer,
  title   = {Longformer: The Long-Document Transformer},
  author  = {Beltagy, Iz and Peters, Matthew E. and Cohan, Arman},
  journal = {arXiv preprint arXiv:2004.05150},
  year    = {2020},
  eprint  = {2004.05150},
  archivePrefix = {arXiv}
}

A Mathematical Framework for Transformer Circuits — Elhage et al. Anthropic, 2021.

@article{elhage2021mathematical,
  title   = {A Mathematical Framework for Transformer Circuits},
  author  = {Elhage, Nelson and Nanda, Neel and Olsson, Catherine and Henighan, Tom and others},
  journal = {Transformer Circuits Thread},
  year    = {2021},
  url     = {https://transformer-circuits.pub/2021/framework/index.html}
}

The Illustrated Transformer — Jay Alammar. 2018.

The Annotated Transformer — Rush et al. Harvard NLP.

Let's build GPT: from scratch, in code, spelled out — Andrej Karpathy. 2023.

Speech and Language Processing — Jurafsky & Martin. 3rd edition draft.

@book{jurafsky2024speech,
  title   = {Speech and Language Processing},
  author  = {Jurafsky, Daniel and Martin, James H.},
  edition = {3rd edition draft},
  year    = {2024},
  url     = {https://web.stanford.edu/~jurafsky/slp3/}
}