bib.bib

@inproceedings{gao-etal-2022-parameter,
  title     = {Parameter-Efficient Mixture-of-Experts Architecture for Pre-trained Language Models},
  author    = {Gao, Ze-Feng  and
               Liu, Peiyu  and
               Zhao, Wayne Xin  and
               Lu, Zhong-Yi  and
               Wen, Ji-Rong},
  editor    = {Calzolari, Nicoletta  and
               Huang, Chu-Ren  and
               Kim, Hansaem  and
               Pustejovsky, James  and
               Wanner, Leo  and
               Choi, Key-Sun  and
               Ryu, Pum-Mo  and
               Chen, Hsin-Hsi  and
               Donatelli, Lucia  and
               Ji, Heng  and
               Kurohashi, Sadao  and
               Paggio, Patrizia  and
               Xue, Nianwen  and
               Kim, Seokhwan  and
               Hahm, Younggyun  and
               He, Zhong  and
               Lee, Tony Kyungil  and
               Santus, Enrico  and
               Bond, Francis  and
               Na, Seung-Hoon},
  booktitle = {Proceedings of the 29th International Conference on Computational Linguistics},
  month     = oct,
  year      = {2022},
  address   = {Gyeongju, Republic of Korea},
  publisher = {International Committee on Computational Linguistics},
  url       = {https://aclanthology.org/2022.coling-1.288/},
  pages     = {3263--3273},
  abstract  = {Recently, Mixture-of-Experts (short as MoE) architecture has achieved remarkable success in increasing the model capacity of large-scale language models. However, MoE requires incorporating significantly more parameters than the base model being extended. In this paper, we propose building a parameter-efficient MoE architecture by sharing information across experts. We adopt matrix product operator (MPO, a tensor decomposition from quantum many-body physics) to reconstruct the parameter matrix in the expert layer and increase model capacity for pre-trained language models by sharing parameters of the central tensor (containing the core information) among different experts while enabling the specificity through the auxiliary tensors (complementing the central tensor) of different experts. To address the unbalanced optimization issue, we further design the gradient mask strategy for the MPO-based MoE architecture. Extensive experiments based on T5 and GPT-2 show improved performance and efficiency of the pre-trained language model (27.2x reduction in total parameters for the superior model performance, compared with the Switch Transformers). Our code is publicly available at \url{https://github.com/RUCAIBox/MPO/MPOE}.}
}

@misc{yi2025edgemoeempoweringsparselarge,
  title         = {EdgeMoE: Empowering Sparse Large Language Models on Mobile Devices},
  author        = {Rongjie Yi and Liwei Guo and Shiyun Wei and Ao Zhou and Shangguang Wang and Mengwei Xu},
  year          = {2025},
  eprint        = {2308.14352},
  archiveprefix = {arXiv},
  primaryclass  = {cs.LG},
  url           = {https://arxiv.org/abs/2308.14352}
}

@inproceedings{li2025moesvd,
  title     = {MoE-{SVD}: Structured Mixture-of-Experts {LLM}s Compression via Singular Value Decomposition},
  author    = {Wei Li and Lujun Li and Hao Gu and You-Liang Huang and Mark G. Lee and Shengjie Sun and Wei Xue and Yike Guo},
  booktitle = {Forty-second International Conference on Machine Learning},
  year      = {2025},
  url       = {https://openreview.net/forum?id=acJ3vdFljk}
}

@inproceedings{10.1145/3669940.3707267,
  author    = {Cao, Shiyi and Liu, Shu and Griggs, Tyler and Schafhalter, Peter and Liu, Xiaoxuan and Sheng, Ying and Gonzalez, Joseph E. and Zaharia, Matei and Stoica, Ion},
  title     = {MoE-Lightning: High-Throughput MoE Inference on Memory-constrained GPUs},
  year      = {2025},
  isbn      = {9798400706981},
  publisher = {Association for Computing Machinery},
  address   = {New York, NY, USA},
  url       = {https://doi.org/10.1145/3669940.3707267},
  doi       = {10.1145/3669940.3707267},
  abstract  = {Efficient deployment of large language models, particularly Mixture of Experts (MoE) models, on resource-constrained platforms presents significant challenges in terms of computational efficiency and memory utilization. The MoE architecture, renowned for its ability to increase model capacity without a proportional increase in inference cost, greatly reduces the token generation latency compared with dense models. However, the large model size makes MoE models inaccessible to individuals without high-end GPUs. In this paper, we propose a high-throughput MoE batch inference system, MoE-Lightning, that significantly outperforms past work. MoE-Lightning introduces a novel CPU-GPU-I/O pipelining schedule, CGOPipe, with paged weights to achieve high resource utilization, and a performance model, HRM, based on a Hierarchical Roofline Model we introduce to help find policies with higher throughput than existing systems. MoE-Lightning can achieve up to (10.3x) higher throughput than state-of-the-art offloading-enabled LLM inference systems for Mixtral 8x7B on a single T4 GPU (16GB). When the theoretical system throughput is bounded by the GPU memory, MoE-Lightning can reach the throughput upper bound with 2-3x less CPU memory, significantly increasing resource utilization. MoE-Lightning also supports efficient batch inference for much larger MoEs (e.g., Mixtral 8x22B and DBRX) on multiple low-cost GPUs (e.g., 2--4 T4s).},
  booktitle = {Proceedings of the 30th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 1},
  pages     = {715–730},
  numpages  = {16},
  keywords  = {batch inference, cpu offloading, moe},
  location  = {Rotterdam, Netherlands},
  series    = {ASPLOS '25}
}

@misc{zhou2022mixtureofexpertsexpertchoicerouting,
  title         = {Mixture-of-Experts with Expert Choice Routing},
  author        = {Yanqi Zhou and Tao Lei and Hanxiao Liu and Nan Du and Yanping Huang and Vincent Zhao and Andrew Dai and Zhifeng Chen and Quoc Le and James Laudon},
  year          = {2022},
  eprint        = {2202.09368},
  archiveprefix = {arXiv},
  primaryclass  = {cs.LG},
  url           = {https://arxiv.org/abs/2202.09368}
}

@inproceedings{10.1145/3577193.3593704,
  author    = {Singh, Siddharth and Ruwase, Olatunji and Awan, Ammar Ahmad and Rajbhandari, Samyam and He, Yuxiong and Bhatele, Abhinav},
  title     = {A Hybrid Tensor-Expert-Data Parallelism Approach to Optimize Mixture-of-Experts Training},
  year      = {2023},
  isbn      = {9798400700569},
  publisher = {Association for Computing Machinery},
  address   = {New York, NY, USA},
  url       = {https://doi.org/10.1145/3577193.3593704},
  doi       = {10.1145/3577193.3593704},
  abstract  = {Mixture-of-Experts (MoE) is a neural network architecture that adds sparsely activated expert blocks to a base model, increasing the number of parameters without impacting computational costs. However, current distributed deep learning frameworks are limited in their ability to train high-quality MoE models with large base models. In this work, we present DeepSpeed-TED, a novel, three-dimensional, hybrid parallel algorithm that combines data, tensor, and expert parallelism to enable the training of MoE models with 4--8\texttimes{} larger base models than the current state-of-the-art. We also describe memory optimizations in the optimizer step, and communication optimizations that eliminate unnecessary data movement. We implement our approach in DeepSpeed and achieve speedups of 26\% over a baseline (i.e. without our communication optimizations) when training a 40 billion parameter MoE model (6.7 billion base model with 16 experts) on 128 V100 GPUs.},
  booktitle = {Proceedings of the 37th ACM International Conference on Supercomputing},
  pages     = {203–214},
  numpages  = {12},
  keywords  = {parallel deep learning, mixture-of-experts, tensor parallelism, expert parallelism},
  location  = {Orlando, FL, USA},
  series    = {ICS '23}
}
@misc{zhang2023h2oheavyhitteroracleefficient,
  title         = {$H_2O$: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models},
  author        = {Zhenyu Zhang and Ying Sheng and Tianyi Zhou and Tianlong Chen and Lianmin Zheng and Ruisi Cai and Zhao Song and Yuandong Tian and Christopher Ré and Clark Barrett and Zhangyang Wang and Beidi Chen},
  year          = {2023},
  eprint        = {2306.14048},
  archiveprefix = {arXiv},
  primaryclass  = {cs.LG},
  url           = {https://arxiv.org/abs/2306.14048}
}
@misc{xiao2024infllmtrainingfreelongcontextextrapolation,
  title         = {InfLLM: Training-Free Long-Context Extrapolation for LLMs with an Efficient Context Memory},
  author        = {Chaojun Xiao and Pengle Zhang and Xu Han and Guangxuan Xiao and Yankai Lin and Zhengyan Zhang and Zhiyuan Liu and Maosong Sun},
  year          = {2024},
  eprint        = {2402.04617},
  archiveprefix = {arXiv},
  primaryclass  = {cs.CL},
  url           = {https://arxiv.org/abs/2402.04617}
}
@misc{hooper2025kvquant10millioncontext,
  title         = {KVQuant: Towards 10 Million Context Length LLM Inference with KV Cache Quantization},
  author        = {Coleman Hooper and Sehoon Kim and Hiva Mohammadzadeh and Michael W. Mahoney and Yakun Sophia Shao and Kurt Keutzer and Amir Gholami},
  year          = {2025},
  eprint        = {2401.18079},
  archiveprefix = {arXiv},
  primaryclass  = {cs.LG},
  url           = {https://arxiv.org/abs/2401.18079}
}
@misc{sun2025shadowkvkvcacheshadows,
  title         = {ShadowKV: KV Cache in Shadows for High-Throughput Long-Context LLM Inference},
  author        = {Hanshi Sun and Li-Wen Chang and Wenlei Bao and Size Zheng and Ningxin Zheng and Xin Liu and Harry Dong and Yuejie Chi and Beidi Chen},
  year          = {2025},
  eprint        = {2410.21465},
  archiveprefix = {arXiv},
  primaryclass  = {cs.LG},
  url           = {https://arxiv.org/abs/2410.21465}
}
@misc{sheng2023flexgenhighthroughputgenerativeinference,
  title         = {FlexGen: High-Throughput Generative Inference of Large Language Models with a Single GPU},
  author        = {Ying Sheng and Lianmin Zheng and Binhang Yuan and Zhuohan Li and Max Ryabinin and Daniel Y. Fu and Zhiqiang Xie and Beidi Chen and Clark Barrett and Joseph E. Gonzalez and Percy Liang and Christopher Ré and Ion Stoica and Ce Zhang},
  year          = {2023},
  eprint        = {2303.06865},
  archiveprefix = {arXiv},
  primaryclass  = {cs.LG},
  url           = {https://arxiv.org/abs/2303.06865}
}
@misc{lee2024infinigenefficientgenerativeinference,
  title         = {InfiniGen: Efficient Generative Inference of Large Language Models with Dynamic KV Cache Management},
  author        = {Wonbeom Lee and Jungi Lee and Junghwan Seo and Jaewoong Sim},
  year          = {2024},
  eprint        = {2406.19707},
  archiveprefix = {arXiv},
  primaryclass  = {cs.LG},
  url           = {https://arxiv.org/abs/2406.19707}
}