References

Evaluation of Adaptive Mixtures of Competing Experts — Nowlan & Hinton. NeurIPS 1990.

@inproceedings{nowlan1990evaluation,
  title     = {Evaluation of Adaptive Mixtures of Competing Experts},
  author    = {Nowlan, Steven J. and Hinton, Geoffrey E.},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  volume    = {3},
  year      = {1990}
}

Adaptive Mixtures of Local Experts — Jacobs, Jordan, Nowlan & Hinton. Neural Computation 3(1), 1991.

@article{jacobs1991adaptive,
  title   = {Adaptive Mixtures of Local Experts},
  author  = {Jacobs, Robert A. and Jordan, Michael I. and Nowlan, Steven J. and Hinton, Geoffrey E.},
  journal = {Neural Computation},
  volume  = {3},
  number  = {1},
  pages   = {79--87},
  year    = {1991}
}

Mixtures-of-Experts — Robert Jacobs. Course handout, University of Rochester, 2008.

@misc{jacobs2008mixtures,
  title        = {Mixtures-of-Experts},
  author       = {Jacobs, Robert A.},
  howpublished = {Handout, Department of Brain and Cognitive Sciences, University of Rochester},
  year         = {2008},
  url          = {https://www2.bcs.rochester.edu/sites/jacobslab/cheat_sheet/mixture_experts.pdf}
}

Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer — Shazeer et al. ICLR 2017.

@inproceedings{shazeer2017outrageously,
  title     = {Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer},
  author    = {Shazeer, Noam and Mirhoseini, Azalia and Maziarz, Krzysztof and Davis, Andy
               and Le, Quoc and Hinton, Geoffrey and Dean, Jeff},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year      = {2017},
  eprint    = {1701.06538},
  archivePrefix = {arXiv}
}

GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding — Lepikhin et al. 2020.

@article{lepikhin2020gshard,
  title   = {GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding},
  author  = {Lepikhin, Dmitry and Lee, HyoukJoong and Xu, Yuanzhong and Chen, Dehao and Firat, Orhan
             and Huang, Yanping and Krikun, Maxim and Shazeer, Noam and Chen, Zhifeng},
  journal = {arXiv preprint arXiv:2006.16668},
  year    = {2020},
  eprint  = {2006.16668},
  archivePrefix = {arXiv}
}

Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity — Fedus, Zoph & Shazeer. JMLR 23, 2022.

@article{fedus2022switch,
  title   = {Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity},
  author  = {Fedus, William and Zoph, Barret and Shazeer, Noam},
  journal = {Journal of Machine Learning Research},
  volume  = {23},
  year    = {2022},
  eprint  = {2101.03961},
  archivePrefix = {arXiv}
}

ST-MoE: Designing Stable and Transferable Sparse Expert Models — Zoph et al. 2022.

@article{zoph2022stmoe,
  title   = {ST-MoE: Designing Stable and Transferable Sparse Expert Models},
  author  = {Zoph, Barret and Bello, Irwan and Kumar, Sameer and Du, Nan and Huang, Yanping
             and Dean, Jeff and Shazeer, Noam and Fedus, William},
  journal = {arXiv preprint arXiv:2202.08906},
  year    = {2022},
  eprint  = {2202.08906},
  archivePrefix = {arXiv}
}

Mixtral of Experts — Jiang et al. 2024.

@article{jiang2024mixtral,
  title   = {Mixtral of Experts},
  author  = {Jiang, Albert Q. and others},
  journal = {arXiv preprint arXiv:2401.04088},
  year    = {2024},
  eprint  = {2401.04088},
  archivePrefix = {arXiv}
}