Publications | Machine Learning & Optimization Lab

Preprints

2026

Sequential Subspace Noise Injection Prevents Accuracy Collapse in Certified Unlearning

Polina Dolgova and Sebastian U. Stich

arXiv preprint arXiv:2601.05134, 2026

@article{dolgova2026sequentialsubspacenoiseinjection,
  title = {Sequential Subspace Noise Injection Prevents Accuracy Collapse in Certified Unlearning},
  author = {Dolgova, Polina and Stich, Sebastian U.},
  journal = {arXiv preprint arXiv:2601.05134},
  year = {2026},
  archiveprefix = {arXiv},
  primaryclass = {cs.LG},
  url = {https://arxiv.org/abs/2601.05134},
}

2025

Monotone Near-Zero-Sum Games: A Generalization of Convex-Concave Minimax

Ruichen Luo, Sebastian U Stich, and K. Chatterjee

arXiv preprint arXiv:2512.02690, 2025

Bib

@article{luo2025monotone,
  author = {Luo, Ruichen and Stich, Sebastian U and Chatterjee, K.},
  title = {Monotone Near-Zero-Sum Games: A Generalization of Convex-Concave Minimax},
  journal = {arXiv preprint arXiv:2512.02690},
  url = {https://arxiv.org/abs/2512.02690},
  year = {2025},
}

Non-Convex Federated Optimization under Cost-Aware Client Selection

Xiaowen Jiang, Anton Rodomanov, and Sebastian U Stich

arXiv preprint arXiv:2512.05327, 2025

Bib

@article{jiang2025non,
  author = {Jiang, Xiaowen and Rodomanov, Anton and Stich, Sebastian U},
  title = {Non-Convex Federated Optimization under Cost-Aware Client Selection},
  journal = {arXiv preprint arXiv:2512.05327},
  url = {https://arxiv.org/abs/2512.05327},
  year = {2025},
}

Composite Optimization with Error Feedback: the Dual Averaging Approach

Yuan Gao, Anton Rodomanov, Jeremy Rack, and Sebastian U. Stich

arXiv preprint arXiv:2510.03507, 2025

Bib

@article{gao2025composite,
  author = {Gao, Yuan and Rodomanov, Anton and Rack, Jeremy and Stich, Sebastian U.},
  title = {Composite Optimization with Error Feedback: the Dual Averaging Approach},
  journal = {arXiv preprint arXiv:2510.03507},
  year = {2025},
  url = {https://arxiv.org/abs/2510.03507},
}

FedMuon: Federated Learning with Bias-corrected LMO-based Optimization

Yuki Takezawa, Anastasia Koloskova, Xiaowen Jiang, and Sebastian U. Stich

arXiv preprint arXiv:2509.26337, 2025

Bib

@article{takezawa2025fedmuon,
  author = {Takezawa, Yuki and Koloskova, Anastasia and Jiang, Xiaowen and Stich, Sebastian U.},
  title = {FedMuon: Federated Learning with Bias-corrected LMO-based Optimization},
  journal = {arXiv preprint arXiv:2509.26337},
  year = {2025},
  url = {https://arxiv.org/abs/2509.26337},
}

Monotone and nonmonotone linearized block coordinate descent methods for nonsmooth composite optimization problems

Yassine Nabou, Lahcen El Bourkhissi, Sebastian U Stich, and Tuomo Valkonen

arXiv preprint arXiv:2506.12397, 2025

Bib

@article{nabou2025monotone,
  author = {Nabou, Yassine and Bourkhissi, Lahcen El and Stich, Sebastian U and Valkonen, Tuomo},
  title = {Monotone and nonmonotone linearized block coordinate descent methods for nonsmooth composite optimization problems},
  journal = {arXiv preprint arXiv:2506.12397},
  url = {https://arxiv.org/abs/2506.12397},
  year = {2025},
}

DADA: Dual Averaging with Distance Adaptation

Mohammad Moshtaghifar, Anton Rodomanov, Daniil Vankov, and Sebastian U. Stich

arXiv preprint arXiv:2501.10258, 2025

Bib

@article{moshtaghifar2025dada,
  author = {Moshtaghifar, Mohammad and Rodomanov, Anton and Vankov, Daniil and Stich, Sebastian U.},
  title = {{DADA:} Dual Averaging with Distance Adaptation},
  journal = {arXiv preprint arXiv:2501.10258},
  year = {2025},
  url = {https://arxiv.org/abs/2501.10258},
}

2023

Robust and Practical Solution of Laplacian Equations by Approximate Elimination

Yuan Gao, Rasmus Kyng, and Daniel A. Spielman

arXiv preprint arXiv:2303.00709, 2023

Bib

@article{gao2023robustpracticalsolutionlaplacian,
  title = {Robust and Practical Solution of Laplacian Equations by Approximate Elimination},
  author = {Gao, Yuan and Kyng, Rasmus and Spielman, Daniel A.},
  journal = {arXiv preprint arXiv:2303.00709},
  year = {2023},
  archiveprefix = {arXiv},
  primaryclass = {math.NA},
  url = {https://arxiv.org/abs/2303.00709},
}

Accepted Papers

2026

AISTATS

Accelerated Distributed Optimization with Compression and Error Feedback

Yuan Gao, Anton Rodomanov, Jeremy Rack, and Sebastian U. Stich

In International Conference on Artificial Intelligence and Statistics (AISTATS), 2026

Bib

@inproceedings{gao2026accelerated,
  author = {Gao, Yuan and Rodomanov, Anton and Rack, Jeremy and Stich, Sebastian U.},
  title = {Accelerated Distributed Optimization with Compression and Error Feedback},
  booktitle = {International Conference on Artificial Intelligence and Statistics (AISTATS)},
  series = {Proceedings of Machine Learning Research},
  volume = {},
  pages = {},
  year = {2026},
  publisher = {{PMLR}},
  url = {https://arxiv.org/abs/2503.08427}
}

2025

NeurIPS

Revisiting Consensus Error: A Fine-grained Analysis of Local SGD under Second-order Data Heterogeneity

Kumar Kshitij Patel, Ali Zindari, Sebastian U Stich, and Linxiao Wang

In Advances in Neural Information Processing Systems (NeurIPS), 2025

Bib

@inproceedings{patel2025revisiting,
  author = {Patel, Kumar Kshitij and Zindari, Ali and Stich, Sebastian U and Wang, Linxiao},
  title = {Revisiting Consensus Error: A Fine-grained Analysis of Local {SGD} under Second-order Data Heterogeneity},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025}
}

A Bias Correction Mechanism for Distributed Asynchronous Optimization

Yuan Gao, Yuki Takezawa, and Sebastian U. Stich

Transactions of Machine Learning Research (TMLR), 2025

Bib

@article{gao2025bias,
  author = {Gao, Yuan and Takezawa, Yuki and Stich, Sebastian U.},
  title = {A Bias Correction Mechanism for Distributed Asynchronous Optimization},
  journal = {Transactions of Machine Learning Research (TMLR)},
  volume = {2025},
  year = {2025},
  url = {https://openreview.net/forum?id=8doMbaah0s}
}

Decentralized gradient tracking with local steps

Yue Liu, Tao Lin, Anastasia Koloskova, and Sebastian U Stich

Optimization Methods and Software, 2025

Bib

@article{liu2025decentralized,
  author = {Liu, Yue and Lin, Tao and Koloskova, Anastasia and Stich, Sebastian U},
  title = {Decentralized gradient tracking with local steps},
  journal = {Optimization Methods and Software},
  pages = {1--28},
  year = {2025}
}

Personalized Federated Learning via Low-Rank Matrix Optimization

Ali Dadras, Sebastian U. Stich, and Alp Yurtsever

Transactions of Machine Learning Research (TMLR), 2025

Bib

@article{dadras2025personalized,
  author = {Dadras, Ali and Stich, Sebastian U. and Yurtsever, Alp},
  title = {Personalized Federated Learning via Low-Rank Matrix Optimization},
  journal = {Transactions of Machine Learning Research (TMLR)},
  volume = {2025},
  year = {2025},
  url = {https://openreview.net/forum?id=DFJu1QB2Nr}
}

ICML

Decoupled SGDA for Games with Intermittent Strategy Communication

Ali Zindari, Parham Yazdkhasti, Anton Rodomanov, Tatjana Chavdarova, and Sebastian U. Stich

In International Conference on Machine Learning (ICML), 2025

Bib

@inproceedings{zindari2025decoupled,
  author = {Zindari, Ali and Yazdkhasti, Parham and Rodomanov, Anton and Chavdarova, Tatjana and Stich, Sebastian U.},
  title = {Decoupled {SGDA} for Games with Intermittent Strategy Communication},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  publisher = {OpenReview.net},
  url = {https://openreview.net/forum?id=ZYkFTSEZ6k}
}

ICML

Yuki Takezawa, Xiaowen Jiang, Anton Rodomanov, and Sebastian U. Stich

In International Conference on Machine Learning (ICML), 2025

Bib

@inproceedings{takezawa2025exploiting,
  author = {Takezawa, Yuki and Jiang, Xiaowen and Rodomanov, Anton and Stich, Sebastian U.},
  title = {Exploiting Similarity for Computation and Communication-Efficient
                    Decentralized Optimization},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  publisher = {OpenReview.net},
  url = {https://openreview.net/forum?id=Td2JZVLitV}
}

AISTATS

Revisiting LocalSGD and SCAFFOLD: Improved Rates and Missing Analysis

Ruichen Luo, Sebastian U. Stich, Samuel Horváth, and Martin Takác

In International Conference on Artificial Intelligence and Statistics (AISTATS), 2025

Bib

@inproceedings{luo2025revisiting,
  author = {Luo, Ruichen and Stich, Sebastian U. and Horv{\'{a}}th, Samuel and Tak{\'{a}}c, Martin},
  title = {Revisiting LocalSGD and {SCAFFOLD:} Improved Rates and Missing Analysis},
  booktitle = {International Conference on Artificial Intelligence and Statistics (AISTATS)},
  series = {Proceedings of Machine Learning Research},
  volume = {258},
  pages = {2539--2547},
  year = {2025},
  publisher = {{PMLR}},
  url = {https://proceedings.mlr.press/v258/luo25c.html}
}

Breaking the Likelihood–Quality Trade-off in Diffusion Models by Merging Pretrained Experts

Yasin Esfandiari, Stefan Bauer, Sebastian U Stich, and Andrea Dittadi

In Workshop on Deep Generative Model in Machine Learning: Theory, Principle and Efficacy @ ICLR, 2025

Bib

@inproceedings{esfandiari2025breaking,
  author = {Esfandiari, Yasin and Bauer, Stefan and Stich, Sebastian U and Dittadi, Andrea},
  title = {Breaking the Likelihood--Quality Trade-off in Diffusion Models by Merging Pretrained Experts},
  booktitle = {Workshop on Deep Generative Model in Machine Learning: Theory, Principle and Efficacy @ ICLR},
  year = {2025}
}

ICLR

Optimizing (L0, L1)-Smooth Functions by Gradient Methods

Daniil Vankov, Anton Rodomanov, Angelia Nedich, Lalitha Sankar, and Sebastian U. Stich

In International Conference on Learning Representations (ICLR), 2025

Bib

@inproceedings{vankov2025optimizing,
  author = {Vankov, Daniil and Rodomanov, Anton and Nedich, Angelia and Sankar, Lalitha and Stich, Sebastian U.},
  title = {Optimizing (L0, L1)-Smooth Functions by Gradient Methods},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2025},
  publisher = {OpenReview.net},
  url = {https://openreview.net/forum?id=GQ1Tc3vHbt}
}

ICLR

Scalable Decentralized Learning with Teleportation

Yuki Takezawa and Sebastian U. Stich

In International Conference on Learning Representations (ICLR), 2025

Bib

@inproceedings{takezawa2025scalable,
  author = {Takezawa, Yuki and Stich, Sebastian U.},
  title = {Scalable Decentralized Learning with Teleportation},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2025},
  publisher = {OpenReview.net},
  url = {https://openreview.net/forum?id=AvmBgiQxxp}
}

ICLR

Towards Faster Decentralized Stochastic Optimization with Communication Compression

Rustem Islamov, Yuan Gao, and Sebastian U. Stich

In International Conference on Learning Representations (ICLR), 2025

Bib

@inproceedings{islamov2025faster,
  author = {Islamov, Rustem and Gao, Yuan and Stich, Sebastian U.},
  title = {Towards Faster Decentralized Stochastic Optimization with Communication
                    Compression},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2025},
  publisher = {OpenReview.net},
  url = {https://openreview.net/forum?id=CMMpcs9prj}
}

LoRAM: Low-Rank Adaptation of Large Language Models on Manifold

Xiaowen Jiang, Xun Wang, and Sebastian U Stich

In Workshop on Sparsity in LLMs (SLLM) @ ICLR, 2025

Bib

@inproceedings{jiang2025loram,
  author = {Jiang, Xiaowen and Wang, Xun and Stich, Sebastian U},
  title = {{LoRAM}: Low-Rank Adaptation of Large Language Models on Manifold},
  booktitle = {Workshop on Sparsity in LLMs (SLLM) @ ICLR},
  year = {2025}
}

2024

Personalized Federated Learning via Low-Rank Matrix Factorization

Ali Dadras, Sebastian U Stich, and Alp Yurtsever

In Workshop on Optimization for Machine Learning @ NeurIPS, 2024

Bib

@inproceedings{dadras2024personalized,
  author = {Dadras, Ali and Stich, Sebastian U and Yurtsever, Alp},
  title = {Personalized Federated Learning via Low-Rank Matrix Factorization},
  booktitle = {Workshop on Optimization for Machine Learning @ NeurIPS},
  year = {2024}
}

NeurIPS

Stabilized Proximal-Point Methods for Federated Optimization

Xiaowen Jiang, Anton Rodomanov, and Sebastian U. Stich

In Advances in Neural Information Processing Systems (NeurIPS), 2024

Spotlight Bib

@inproceedings{jiang2024stabilized,
  author = {Jiang, Xiaowen and Rodomanov, Anton and Stich, Sebastian U.},
  title = {Stabilized Proximal-Point Methods for Federated Optimization},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://arxiv.org/abs/2407.07084},
}

NeurIPS

Universality of AdaGrad Stepsizes for Stochastic Optimization: Inexact Oracle, Acceleration and Variance Reduction

Anton Rodomanov, Xiaowen Jiang, and Sebastian U. Stich

In Advances in Neural Information Processing Systems (NeurIPS), 2024

Bib

@inproceedings{rodomanov2024universality,
  author = {Rodomanov, Anton and Jiang, Xiaowen and Stich, Sebastian U.},
  title = {Universality of AdaGrad Stepsizes for Stochastic Optimization: Inexact
                    Oracle, Acceleration and Variance Reduction},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://arxiv.org/abs/2406.06398}
}

DADA: Dual Averaging with Distance Adaptation

Mohammad Moshtaghifar, Anton Rodomanov, Daniil Vankov, and Sebastian U Stich

In Workshop on Optimization for Machine Learning @ NeurIPS, 2024

Bib

@inproceedings{moshtaghifar2024dada,
  author = {Moshtaghifar, Mohammad and Rodomanov, Anton and Vankov, Daniil and Stich, Sebastian U},
  title = {{DADA}: Dual Averaging with Distance Adaptation},
  booktitle = {Workshop on Optimization for Machine Learning @ NeurIPS},
  year = {2024}
}

NeurIPS

Hierarchical uncertainty exploration via feedforward posterior trees

Elias Nehme, Rotem Mulayoff, and Tomer Michaeli

In Advances in Neural Information Processing Systems (NeurIPS), 2024

Bib

@inproceedings{nehme2024uncertainty,
  author = {Nehme, Elias and Mulayoff, Rotem and Michaeli, Tomer},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  title = {Hierarchical uncertainty exploration via feedforward posterior trees},
  year = {2024},
  url = {https://arxiv.org/abs/2405.15719}
}

Locally Adaptive Federated Learning

Sohom Mukherjee, Nicolas Loizou, and Sebastian U. Stich

Transactions of Machine Learning Research (TMLR), 2024

Bib

@article{mukherjee2024locally,
  author = {Mukherjee, Sohom and Loizou, Nicolas and Stich, Sebastian U.},
  title = {Locally Adaptive Federated Learning},
  journal = {Transactions of Machine Learning Research (TMLR)},
  volume = {2024},
  year = {2024},
  url = {https://openreview.net/forum?id=auLdS1iuKW}
}

Decoupled Stochastic Gradient Descent for N-Player Games

Ali Zindari, Parham Yazdkhasti, A Rodomanov, Tatjana Chavdarova, and Sebastian U Stich

In Workshop on Aligning Reinforcement Learning Experimentalists and Theorists @ ICML, 2024

Bib

@inproceedings{zindari2024decoupled,
  author = {Zindari, Ali and Yazdkhasti, Parham and Rodomanov, A and Chavdarova, Tatjana and Stich, Sebastian U},
  title = {Decoupled Stochastic Gradient Descent for $N$-Player Games},
  booktitle = {Workshop on Aligning Reinforcement Learning Experimentalists and Theorists @ ICML},
  year = {2024}
}

ICML

Federated Optimization with Doubly Regularized Drift Correction

Xiaowen Jiang, Anton Rodomanov, and Sebastian U. Stich

In International Conference on Machine Learning (ICML), 2024

Bib

@inproceedings{jiang2024federated,
  author = {Jiang, Xiaowen and Rodomanov, Anton and Stich, Sebastian U.},
  title = {Federated Optimization with Doubly Regularized Drift Correction},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  publisher = {OpenReview.net},
  url = {https://openreview.net/forum?id=JD03zxWZzs}
}

ICML

Non-convex Stochastic Composite Optimization with Polyak Momentum

Yuan Gao, Anton Rodomanov, and Sebastian U. Stich

In International Conference on Machine Learning (ICML), 2024

Bib

@inproceedings{gao2024non,
  author = {Gao, Yuan and Rodomanov, Anton and Stich, Sebastian U.},
  title = {Non-convex Stochastic Composite Optimization with Polyak Momentum},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  publisher = {OpenReview.net},
  url = {https://openreview.net/forum?id=1ySQI9LE4w}
}

ICML

On Convergence of Incremental Gradient for Non-convex Smooth Functions

Anastasia Koloskova, Nikita Doikov, Sebastian U. Stich, and Martin Jaggi

In International Conference on Machine Learning (ICML), 2024

Bib

@inproceedings{koloskova2024convergence,
  author = {Koloskova, Anastasia and Doikov, Nikita and Stich, Sebastian U. and Jaggi, Martin},
  title = {On Convergence of Incremental Gradient for Non-convex Smooth Functions},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  publisher = {OpenReview.net},
  url = {https://openreview.net/forum?id=ZRMQX6aTUS}
}

ICML

Spectral Preconditioning for Gradient Methods on Graded Non-convex Functions

Nikita Doikov, Sebastian U. Stich, and Martin Jaggi

In International Conference on Machine Learning (ICML), 2024

Bib

@inproceedings{doikov2024spectral,
  author = {Doikov, Nikita and Stich, Sebastian U. and Jaggi, Martin},
  title = {Spectral Preconditioning for Gradient Methods on Graded Non-convex
                    Functions},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  publisher = {OpenReview.net},
  url = {https://openreview.net/forum?id=NvBJOcmti6}
}

COLT

The Limits and Potentials of Local SGD for Distributed Heterogeneous Learning with Intermittent Communication

Kumar Kshitij Patel, Margalit Glasgow, Ali Zindari, Lingxiao Wang, Sebastian U. Stich, Ziheng Cheng, Nirmit Joshi, and Nathan Srebro

In Conference on Learning Theory (COLT), 2024

Bib

@inproceedings{patel2024limits,
  author = {Patel, Kumar Kshitij and Glasgow, Margalit and Zindari, Ali and Wang, Lingxiao and Stich, Sebastian U. and Cheng, Ziheng and Joshi, Nirmit and Srebro, Nathan},
  title = {The Limits and Potentials of Local {SGD} for Distributed Heterogeneous
                    Learning with Intermittent Communication},
  booktitle = {Conference on Learning Theory (COLT)},
  series = {Proceedings of Machine Learning Research},
  volume = {247},
  pages = {4115--4157},
  year = {2024},
  publisher = {{PMLR}},
  url = {https://proceedings.mlr.press/v247/patel24a.html}
}

ICLR

An improved analysis of per-sample and per-update clipping in federated learning

Bo Li, Xiaowen Jiang, Mikkel N. Schmidt, Tommy Sonne Alstrøm, and Sebastian U. Stich

In International Conference on Learning Representations (ICLR), 2024

Bib

@inproceedings{li2024improved,
  author = {Li, Bo and Jiang, Xiaowen and Schmidt, Mikkel N. and Alstr{\o}m, Tommy Sonne and Stich, Sebastian U.},
  title = {An improved analysis of per-sample and per-update clipping in federated
                    learning},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  publisher = {OpenReview.net},
  url = {https://openreview.net/forum?id=BdPvGRvoBC}
}

ICLR

Communication-Efficient Gradient Descent-Accent Methods for Distributed Variational Inequalities: Unified Analysis and Local Updates

Siqi Zhang, Sayantan Choudhury, Sebastian U. Stich, and Nicolas Loizou

In International Conference on Learning Representations (ICLR), 2024

Bib

@inproceedings{zhang2024communication,
  author = {Zhang, Siqi and Choudhury, Sayantan and Stich, Sebastian U. and Loizou, Nicolas},
  title = {Communication-Efficient Gradient Descent-Accent Methods for Distributed
                    Variational Inequalities: Unified Analysis and Local Updates},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  publisher = {OpenReview.net},
  url = {https://openreview.net/forum?id=hORCalGn3Z}
}

ICLR

EControl: Fast Distributed Optimization with Compression and Error Control

Yuan Gao, Rustem Islamov, and Sebastian U. Stich

In International Conference on Learning Representations (ICLR), 2024

Bib

@inproceedings{gao2024econtrol,
  author = {Gao, Yuan and Islamov, Rustem and Stich, Sebastian U.},
  title = {EControl: Fast Distributed Optimization with Compression and Error
                    Control},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  publisher = {OpenReview.net},
  url = {https://openreview.net/forum?id=lsvlvWB9vz}
}

Global complexity analysis of BFGS

Anton Rodomanov

arXiv preprint arXiv:2404.15051, 2024

Bib

@article{rodomanov2024bfgs,
  author = {Rodomanov, Anton},
  journal = {arXiv preprint arXiv:2404.15051},
  title = {Global complexity analysis of {BFGS}},
  year = {2024},
  url = {https://arxiv.org/abs/2404.15051}
}

Synthetic data shuffling accelerates the convergence of federated learning under data heterogeneity

Bo Li, Yasin Esfandiari, Mikkel N. Schmidt, Tommy Sonne Alstrøm, and Sebastian U. Stich

Transactions of Machine Learning Research (TMLR), 2024

Bib

@article{li2024synthetic,
  author = {Li, Bo and Esfandiari, Yasin and Schmidt, Mikkel N. and Alstr{\o}m, Tommy Sonne and Stich, Sebastian U.},
  title = {Synthetic data shuffling accelerates the convergence of federated
                    learning under data heterogeneity},
  journal = {Transactions of Machine Learning Research (TMLR)},
  volume = {2024},
  year = {2024},
  url = {https://openreview.net/forum?id=c5o4HUypqm}
}

2023

NeurIPS

Adaptive SGD with Polyak stepsize and Line-search: Robust Convergence and Variance Reduction

Xiaowen Jiang and Sebastian U. Stich

In Advances in Neural Information Processing Systems (NeurIPS), 2023

Bib

@inproceedings{jiang2023adaptive,
  author = {Jiang, Xiaowen and Stich, Sebastian U.},
  title = {Adaptive {SGD} with Polyak stepsize and Line-search: Robust Convergence
                    and Variance Reduction},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2023},
  url = {https://arxiv.org/abs/2308.06058}
}

Diversity-adjusted adaptive step size

Parham Yazdkhasti, Xiaowen Jiang, and Sebastian U Stich

In Workshop on Optimization for Machine Learning @ NeurIPS, 2023

Bib

@inproceedings{yazdkhasti2023diversity,
  author = {Yazdkhasti, Parham and Jiang, Xiaowen and Stich, Sebastian U},
  title = {Diversity-adjusted adaptive step size},
  booktitle = {Workshop on Optimization for Machine Learning @ NeurIPS},
  year = {2023}
}

Noise Injection Irons Out Local Minima and Saddle Points

Konstantin Mishchenko and Sebastian U Stich

In Workshop on Optimization for Machine Learning @ NeurIPS, 2023

Bib

@inproceedings{mishchenko2023noise,
  author = {Mishchenko, Konstantin and Stich, Sebastian U},
  title = {Noise Injection Irons Out Local Minima and Saddle Points},
  booktitle = {Workshop on Optimization for Machine Learning @ NeurIPS},
  year = {2023}
}

On the Convergence of Local SGD Under Third-Order Smoothness and Hessian Similarity

Ali Zindari, Ruichen Luo, and Sebastian U Stich

In Workshop on Optimization for Machine Learning @ NeurIPS, 2023

Bib

@inproceedings{zindari2023convergence,
  author = {Zindari, Ali and Luo, Ruichen and Stich, Sebastian U},
  title = {On the Convergence of Local {SGD} Under Third-Order Smoothness and Hessian Similarity},
  booktitle = {Workshop on Optimization for Machine Learning @ NeurIPS},
  year = {2023}
}

ICML

Revisiting Gradient Clipping: Stochastic bias and tight convergence guarantees

Anastasia Koloskova, Hadrien Hendrikx, and Sebastian U. Stich

In International Conference on Machine Learning (ICML), 2023

Bib

@inproceedings{koloskova2023revisiting,
  author = {Koloskova, Anastasia and Hendrikx, Hadrien and Stich, Sebastian U.},
  title = {Revisiting Gradient Clipping: Stochastic bias and tight convergence
                    guarantees},
  booktitle = {International Conference on Machine Learning (ICML)},
  series = {Proceedings of Machine Learning Research},
  volume = {202},
  pages = {17343--17363},
  year = {2023},
  publisher = {{PMLR}},
  url = {https://proceedings.mlr.press/v202/koloskova23a.html}
}

ICML

Special Properties of Gradient Descent with Large Learning Rates

Amirkeivan Mohtashami, Martin Jaggi, and Sebastian U. Stich

In International Conference on Machine Learning (ICML), 2023

Bib

@inproceedings{mohtashami2023special,
  author = {Mohtashami, Amirkeivan and Jaggi, Martin and Stich, Sebastian U.},
  title = {Special Properties of Gradient Descent with Large Learning Rates},
  booktitle = {International Conference on Machine Learning (ICML)},
  series = {Proceedings of Machine Learning Research},
  volume = {202},
  pages = {25082--25104},
  year = {2023},
  publisher = {{PMLR}},
  url = {https://proceedings.mlr.press/v202/mohtashami23a.html}
}

CVPR

On the Effectiveness of Partial Variance Reduction in Federated Learning with Heterogeneous Data

Bo Li, Mikkel N. Schmidt, Tommy S. Alstrøm, and Sebastian U. Stich

In IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), 2023

Bib

@inproceedings{li2023effectiveness,
  author = {Li, Bo and Schmidt, Mikkel N. and Alstr{\o}m, Tommy S. and Stich, Sebastian U.},
  title = {On the Effectiveness of Partial Variance Reduction in Federated Learning
                    with Heterogeneous Data},
  booktitle = {{IEEE/CVF} Conference on Computer Vision and Pattern Recognition (CVPR)},
  pages = {3964--3973},
  year = {2023},
  publisher = {{IEEE}},
  url = {https://arxiv.org/abs/2212.02191}
}

Stochastic distributed learning with gradient quantization and double-variance reduction

Samuel Horváth, Dmitry Kovalev, Konstantin Mishchenko, Peter Richtárik, and Sebastian U. Stich

Optimization Methods and Software, 2023

Bib

@article{horvath2023stochastic,
  author = {Horv{\'{a}}th, Samuel and Kovalev, Dmitry and Mishchenko, Konstantin and Richt{\'{a}}rik, Peter and Stich, Sebastian U.},
  title = {Stochastic distributed learning with gradient quantization and double-variance
                    reduction},
  journal = {Optimization Methods and Software},
  volume = {38},
  number = {1},
  pages = {91--106},
  year = {2023},
  url = {https://arxiv.org/abs/1904.05115}
}

2022

Partial Variance Reduction improves Non-Convex Federated learning on heterogeneous data

Bo Li, Mikkel N. Schmidt, Tommy S. Alstrøm, and Sebastian U. Stich

arXiv preprint arXiv:2212.02191, 2022

Bib

@article{li2022partial,
  author = {Li, Bo and Schmidt, Mikkel N. and Alstr{\o}m, Tommy S. and Stich, Sebastian U.},
  title = {Partial Variance Reduction improves Non-Convex Federated learning
                    on heterogeneous data},
  journal = {arXiv preprint arXiv:2212.02191},
  year = {2022},
  url = {https://arxiv.org/abs/2212.02191},
}

Bidirectional Adaptive Communication for Heterogeneous Distributed Learning

Dmitrii Avdiukhin, Vladimir Braverman, Nikita Ivkin, and Sebastian U Stich

In Workshop on Optimization for Machine Learning @ NeurIPS, 2022

Bib

@inproceedings{avdiukhin2022bidirectional,
  author = {Avdiukhin, Dmitrii and Braverman, Vladimir and Ivkin, Nikita and Stich, Sebastian U},
  title = {Bidirectional Adaptive Communication for Heterogeneous Distributed Learning},
  booktitle = {Workshop on Optimization for Machine Learning @ NeurIPS},
  year = {2022}
}

Data-heterogeneity-aware mixing for decentralized learning

Yatin Dandi, Anastasia Koloskova, Martin Jaggi, and Sebastian U Stich

In Workshop on Optimization for Machine Learning @ NeurIPS, 2022

Bib

@inproceedings{dandi2022data,
  author = {Dandi, Yatin and Koloskova, Anastasia and Jaggi, Martin and Stich, Sebastian U},
  title = {Data-heterogeneity-aware mixing for decentralized learning},
  booktitle = {Workshop on Optimization for Machine Learning @ NeurIPS},
  year = {2022}
}

NeurIPS

Decentralized Local Stochastic Extra-Gradient for Variational Inequalities

Aleksandr Beznosikov, Pavel E. Dvurechensky, Anastasia Koloskova, Valentin Samokhin, Sebastian U. Stich, and Alexander V. Gasnikov

In Advances in Neural Information Processing Systems (NeurIPS), 2022

Bib

@inproceedings{beznosikov2022decentralized,
  author = {Beznosikov, Aleksandr and Dvurechensky, Pavel E. and Koloskova, Anastasia and Samokhin, Valentin and Stich, Sebastian U. and Gasnikov, Alexander V.},
  title = {Decentralized Local Stochastic Extra-Gradient for Variational Inequalities},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2022},
  url = {https://arxiv.org/abs/2106.08315}
}

Preserving privacy with PATE for heterogeneous data

Akshay Dodwadmath and Sebastian U Stich

In Workshop on Distribution Shifts: Connecting Methods and Applications @ NeurIPS, 2022

Bib

@inproceedings{dodwadmath2022preserving,
  author = {Dodwadmath, Akshay and Stich, Sebastian U},
  title = {Preserving privacy with {PATE} for heterogeneous data},
  booktitle = {Workshop on Distribution Shifts: Connecting Methods and Applications @ NeurIPS},
  year = {2022}
}

NeurIPS

Sharper Convergence Guarantees for Asynchronous SGD for Distributed and Federated Learning

Anastasia Koloskova, Sebastian U. Stich, and Martin Jaggi

In Advances in Neural Information Processing Systems (NeurIPS), 2022

Oral Bib

@inproceedings{koloskova2022sharper,
  author = {Koloskova, Anastasia and Stich, Sebastian U. and Jaggi, Martin},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  title = {Sharper Convergence Guarantees for Asynchronous {SGD} for Distributed and Federated Learning},
  year = {2022},
  url = {https://arxiv.org/abs/2206.08307}
}

ICML

ProgFed: Effective, Communication, and Computation Efficient Federated Learning by Progressive Training

Hui-Po Wang, Sebastian U. Stich, Yang He, and Mario Fritz

In International Conference on Machine Learning (ICML), 2022

Bib

@inproceedings{wang2022progfed,
  author = {Wang, Hui{-}Po and Stich, Sebastian U. and He, Yang and Fritz, Mario},
  title = {ProgFed: Effective, Communication, and Computation Efficient Federated
                    Learning by Progressive Training},
  booktitle = {International Conference on Machine Learning (ICML)},
  series = {Proceedings of Machine Learning Research},
  volume = {162},
  pages = {23034--23054},
  year = {2022},
  publisher = {{PMLR}},
  url = {https://proceedings.mlr.press/v162/wang22y.html}
}

ICML

ProxSkip: Yes! Local Gradient Steps Provably Lead to Communication Acceleration! Finally!

Konstantin Mishchenko, Grigory Malinovsky, Sebastian U. Stich, and Peter Richtárik

In International Conference on Machine Learning (ICML), 2022

Bib

@inproceedings{mishchenko2022proxskip,
  author = {Mishchenko, Konstantin and Malinovsky, Grigory and Stich, Sebastian U. and Richt{\'{a}}rik, Peter},
  title = {ProxSkip: Yes! Local Gradient Steps Provably Lead to Communication
                    Acceleration! Finally!},
  booktitle = {International Conference on Machine Learning (ICML)},
  series = {Proceedings of Machine Learning Research},
  volume = {162},
  pages = {15750--15769},
  year = {2022},
  publisher = {{PMLR}},
  url = {https://proceedings.mlr.press/v162/mishchenko22b.html}
}

On Avoiding Local Minima Using Gradient Descent With Large Learning Rates

Amirkeivan Mohtashami, Martin Jaggi, and Sebastian U. Stich

arXiv preprint arXiv:2205.15142, 2022

Bib

@article{mohtashami2022avoiding,
  author = {Mohtashami, Amirkeivan and Jaggi, Martin and Stich, Sebastian U.},
  title = {On Avoiding Local Minima Using Gradient Descent With Large Learning
                    Rates},
  journal = {arXiv preprint arXiv:2205.15142},
  year = {2022},
  url = {https://arxiv.org/abs/2205.15142},
}

AISTATS

Masked Training of Neural Networks with Partial Gradients

Amirkeivan Mohtashami, Martin Jaggi, and Sebastian U. Stich

In International Conference on Artificial Intelligence and Statistics (AISTATS), 2022

Bib

@inproceedings{mohtashami2022masked,
  author = {Mohtashami, Amirkeivan and Jaggi, Martin and Stich, Sebastian U.},
  title = {Masked Training of Neural Networks with Partial Gradients},
  booktitle = {International Conference on Artificial Intelligence and Statistics (AISTATS)},
  series = {Proceedings of Machine Learning Research},
  volume = {151},
  pages = {5876--5890},
  year = {2022},
  publisher = {{PMLR}},
  url = {https://proceedings.mlr.press/v151/mohtashami22a.html}
}

Characterizing & Finding Good Data Orderings for Fast Convergence of Sequential Gradient Methods

Amirkeivan Mohtashami, Sebastian U. Stich, and Martin Jaggi

arXiv preprint arXiv:2202.01838, 2022

Bib

@article{mohtashami2022characterizing,
  author = {Mohtashami, Amirkeivan and Stich, Sebastian U. and Jaggi, Martin},
  title = {Characterizing {\&} Finding Good Data Orderings for Fast Convergence
                    of Sequential Gradient Methods},
  journal = {arXiv preprint arXiv:2202.01838},
  year = {2022},
  url = {https://arxiv.org/abs/2202.01838},
}

2021

NeurIPS

An Improved Analysis of Gradient Tracking for Decentralized Machine Learning

Anastasia Koloskova, Tao Lin, and Sebastian U. Stich

In Advances in Neural Information Processing Systems (NeurIPS), 2021

Bib

@inproceedings{koloskova2021improved,
  author = {Koloskova, Anastasia and Lin, Tao and Stich, Sebastian U.},
  title = {An Improved Analysis of Gradient Tracking for Decentralized Machine
                    Learning},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  pages = {11422--11435},
  year = {2021},
  url = {https://arxiv.org/abs/2202.03836}
}

NeurIPS

Breaking the centralized barrier for cross-device federated learning

Sai Praneeth Karimireddy, Martin Jaggi, Satyen Kale, Mehryar Mohri, Sashank J. Reddi, Sebastian U. Stich, and Ananda Theertha Suresh

In Advances in Neural Information Processing Systems (NeurIPS), 2021

Bib

@inproceedings{karimireddy2021breaking,
  author = {Karimireddy, Sai Praneeth and Jaggi, Martin and Kale, Satyen and Mohri, Mehryar and Reddi, Sashank J. and Stich, Sebastian U. and Suresh, Ananda Theertha},
  title = {Breaking the centralized barrier for cross-device federated learning},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  pages = {28663--28676},
  year = {2021},
  url = {https://openreview.net/forum?id=FMPuzXV1fR}
}

NeurIPS

RelaySum for Decentralized Deep Learning on Heterogeneous Data

Thijs Vogels, Lie He, Anastasia Koloskova, Sai Praneeth Karimireddy, Tao Lin, Sebastian U. Stich, and Martin Jaggi

In Advances in Neural Information Processing Systems (NeurIPS), 2021

Bib

@inproceedings{vogels2021relaysum,
  author = {Vogels, Thijs and He, Lie and Koloskova, Anastasia and Karimireddy, Sai Praneeth and Lin, Tao and Stich, Sebastian U. and Jaggi, Martin},
  title = {RelaySum for Decentralized Deep Learning on Heterogeneous Data},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  pages = {28004--28015},
  year = {2021},
  url = {https://arxiv.org/abs/2110.04175}
}

Tackling benign nonconvexity with smoothing and stochastic gradients

Harsh Vardhan and Sebastian U Stich

In Workshop on Optimization for Machine Learning @ NeurIPS, 2021

Bib

@inproceedings{vardhan2021tackling,
  author = {Vardhan, Harsh and Stich, Sebastian U},
  title = {Tackling benign nonconvexity with smoothing and stochastic gradients},
  booktitle = {Workshop on Optimization for Machine Learning @ NeurIPS},
  year = {2021}
}

The peril of popular deep learning uncertainty estimation methods

Yehao Liu, Matteo Pagliardini, Tatjana Chavdarova, and Sebastian U Stich

In Workshop on Bayesian Deep Learning @ NeurIPS, 2021

Bib

@inproceedings{liu2021peril,
  author = {Liu, Yehao and Pagliardini, Matteo and Chavdarova, Tatjana and Stich, Sebastian U},
  title = {The peril of popular deep learning uncertainty estimation methods},
  booktitle = {Workshop on Bayesian Deep Learning @ NeurIPS},
  year = {2021}
}

ICML

Consensus Control for Decentralized Deep Learning

Lingjing Kong, Tao Lin, Anastasia Koloskova, Martin Jaggi, and Sebastian U. Stich

In International Conference on Machine Learning (ICML), 2021

Bib

@inproceedings{kong2021consensus,
  author = {Kong, Lingjing and Lin, Tao and Koloskova, Anastasia and Jaggi, Martin and Stich, Sebastian U.},
  title = {Consensus Control for Decentralized Deep Learning},
  booktitle = {International Conference on Machine Learning (ICML)},
  series = {Proceedings of Machine Learning Research},
  volume = {139},
  pages = {5686--5696},
  year = {2021},
  publisher = {{PMLR}},
  url = {http://proceedings.mlr.press/v139/kong21a.html}
}

ICML

Quasi-global Momentum: Accelerating Decentralized Deep Learning on Heterogeneous Data

Tao Lin, Sai Praneeth Karimireddy, Sebastian U. Stich, and Martin Jaggi

In International Conference on Machine Learning (ICML), 2021

Bib

@inproceedings{lin2021quasi,
  author = {Lin, Tao and Karimireddy, Sai Praneeth and Stich, Sebastian U. and Jaggi, Martin},
  title = {Quasi-global Momentum: Accelerating Decentralized Deep Learning on
                    Heterogeneous Data},
  booktitle = {International Conference on Machine Learning (ICML)},
  series = {Proceedings of Machine Learning Research},
  volume = {139},
  pages = {6654--6665},
  year = {2021},
  publisher = {{PMLR}},
  url = {http://proceedings.mlr.press/v139/lin21c.html}
}

Advances and Open Problems in Federated Learning

Peter Kairouz, H. Brendan McMahan, Brendan Avent, Aurélien Bellet, Mehdi Bennis, Arjun Nitin Bhagoji, Kallista A. Bonawitz, Zachary Charles, Graham Cormode, Rachel Cummings, Rafael G. L. D’Oliveira, Hubert Eichner, Salim El Rouayheb, David Evans, Josh Gardner, Zachary Garrett, Adrià Gascón, Badih Ghazi, Phillip B. Gibbons, Marco Gruteser, Zaı̈d Harchaoui, Chaoyang He, Lie He, Zhouyuan Huo, Ben Hutchinson, Justin Hsu, Martin Jaggi, Tara Javidi, Gauri Joshi, Mikhail Khodak, Jakub Konečný, Aleksandra Korolova, Farinaz Koushanfar, Sanmi Koyejo, Tancrède Lepoint, Yang Liu, Prateek Mittal, Mehryar Mohri, Richard Nock, Ayfer Özgür, Rasmus Pagh, Hang Qi, Daniel Ramage, Ramesh Raskar, Mariana Raykova, Dawn Song, Weikang Song, Sebastian U. Stich, Ziteng Sun, Ananda Theertha Suresh, Florian Tramèr, Praneeth Vepakomma, Jianyu Wang, Li Xiong, Zheng Xu, Qiang Yang, Felix X. Yu, Han Yu, and Sen Zhao

Foundations and Trends in Machine Learning, 2021

Bib

@article{kairouz2021advances,
  author = {Kairouz, Peter and McMahan, H. Brendan and Avent, Brendan and Bellet, Aur{\'{e}}lien and Bennis, Mehdi and Bhagoji, Arjun Nitin and Bonawitz, Kallista A. and Charles, Zachary and Cormode, Graham and Cummings, Rachel and D'Oliveira, Rafael G. L. and Eichner, Hubert and Rouayheb, Salim El and Evans, David and Gardner, Josh and Garrett, Zachary and Gasc{\'{o}}n, Adri{\`{a}} and Ghazi, Badih and Gibbons, Phillip B. and Gruteser, Marco and Harchaoui, Za{\"{\i}}d and He, Chaoyang and He, Lie and Huo, Zhouyuan and Hutchinson, Ben and Hsu, Justin and Jaggi, Martin and Javidi, Tara and Joshi, Gauri and Khodak, Mikhail and Kone{\v{c}}n{\'y}, Jakub and Korolova, Aleksandra and Koushanfar, Farinaz and Koyejo, Sanmi and Lepoint, Tancr{\`{e}}de and Liu, Yang and Mittal, Prateek and Mohri, Mehryar and Nock, Richard and {\"{O}}zg{\"{u}}r, Ayfer and Pagh, Rasmus and Qi, Hang and Ramage, Daniel and Raskar, Ramesh and Raykova, Mariana and Song, Dawn and Song, Weikang and Stich, Sebastian U. and Sun, Ziteng and Suresh, Ananda Theertha and Tram{\`{e}}r, Florian and Vepakomma, Praneeth and Wang, Jianyu and Xiong, Li and Xu, Zheng and Yang, Qiang and Yu, Felix X. and Yu, Han and Zhao, Sen},
  title = {Advances and Open Problems in Federated Learning},
  journal = {Foundations and Trends in Machine Learning},
  volume = {14},
  number = {1-2},
  pages = {1--210},
  year = {2021},
  url = {https://arxiv.org/abs/1912.04977}
}

AISTATS

A Linearly Convergent Algorithm for Decentralized Optimization: Sending Less Bits for Free!

Dmitry Kovalev, Anastasia Koloskova, Martin Jaggi, Peter Richtárik, and Sebastian U. Stich

In International Conference on Artificial Intelligence and Statistics (AISTATS), 2021

Bib

@inproceedings{kovalev2021linearly,
  author = {Kovalev, Dmitry and Koloskova, Anastasia and Jaggi, Martin and Richt{\'{a}}rik, Peter and Stich, Sebastian U.},
  title = {A Linearly Convergent Algorithm for Decentralized Optimization: Sending
                    Less Bits for Free!},
  booktitle = {International Conference on Artificial Intelligence and Statistics (AISTATS)},
  series = {Proceedings of Machine Learning Research},
  volume = {130},
  pages = {4087--4095},
  year = {2021},
  publisher = {{PMLR}},
  url = {http://proceedings.mlr.press/v130/kovalev21a.html}
}

AISTATS

Critical Parameters for Scalable Distributed Learning with Large Batches and Asynchronous Updates

Sebastian U. Stich, Amirkeivan Mohtashami, and Martin Jaggi

In International Conference on Artificial Intelligence and Statistics (AISTATS), 2021

Bib

@inproceedings{stich2021critical,
  author = {Stich, Sebastian U. and Mohtashami, Amirkeivan and Jaggi, Martin},
  title = {Critical Parameters for Scalable Distributed Learning with Large Batches
                    and Asynchronous Updates},
  booktitle = {International Conference on Artificial Intelligence and Statistics (AISTATS)},
  series = {Proceedings of Machine Learning Research},
  volume = {130},
  pages = {4042--4050},
  year = {2021},
  publisher = {{PMLR}},
  url = {http://proceedings.mlr.press/v130/stich21a.html}
}

AISTATS

LENA: Communication-Efficient Distributed Learning with Self-Triggered Gradient Uploads

Hossein Shokri Ghadikolaei, Sebastian U. Stich, and Martin Jaggi

In International Conference on Artificial Intelligence and Statistics (AISTATS), 2021

Bib

@inproceedings{ghadikolaei2021lena,
  author = {Ghadikolaei, Hossein Shokri and Stich, Sebastian U. and Jaggi, Martin},
  title = {{LENA:} Communication-Efficient Distributed Learning with Self-Triggered
                    Gradient Uploads},
  booktitle = {International Conference on Artificial Intelligence and Statistics (AISTATS)},
  series = {Proceedings of Machine Learning Research},
  volume = {130},
  pages = {3943--3951},
  year = {2021},
  publisher = {{PMLR}},
  url = {http://proceedings.mlr.press/v130/shokri-ghadikolaei21a.html}
}