corpus.blog
41554 blogs · [ { "id": "01a0879e-aebc-7209-be67-74f184763485", "title": "Muon Optimizer Guide - Quick Start and Key Details", "url": "https://main-horse.github.io/translations/kexue/11416/", "published_at": "2025-11-18T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f185543e34", "title": "Asymptotic Estimation of AdamW's Weight RMS (Part 2)", "url": "https://main-horse.github.io/translations/kexue/11404/", "published_at": "2025-11-16T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f185c71656", "title": "Asymptotic Estimate for the Maximum of n Normal Random Variables", "url": "https://main-horse.github.io/translations/kexue/11390/", "published_at": "2025-11-05T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1868bcd37", "title": "5. Dual Gradient Descent", "url": "https://main-horse.github.io/translations/kexue/11388/", "published_at": "2025-11-02T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f186b1cb97", "title": "Low-Precision Attention May Suffer from Biased Rounding Errors", "url": "https://main-horse.github.io/translations/kexue/11371/", "published_at": "2025-10-26T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f186fd7b19", "title": "1. Three Characteristics of a Good Model", "url": "https://main-horse.github.io/translations/kexue/11340/", "published_at": "2025-10-20T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f187bd50c5", "title": "Fast Estimation of the Spectral Norm of Random Matrices", "url": "https://main-horse.github.io/translations/kexue/11335/", "published_at": "2025-10-11T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1887f7098", "title": "A Very Concise VQ Training Scheme", "url": "https://main-horse.github.io/translations/kexue/11328/", "published_at": "2025-10-07T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f189787c16", "title": "Why Add Short Conv to Linear Attention?", "url": "https://main-horse.github.io/translations/kexue/11320/", "published_at": "2025-10-04T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18a6507bf", "title": "Asymptotic Estimation of AdamW's Weight RMS", "url": "https://main-horse.github.io/translations/kexue/11307/", "published_at": "2025-09-30T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18a9d6927", "title": "Rethinking Learning Rate and Batch Size (IV) - EMA", "url": "https://main-horse.github.io/translations/kexue/11301/", "published_at": "2025-09-21T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18aed2081", "title": "Rethinking Learning Rate and Batch Size (III) - Muon", "url": "https://main-horse.github.io/translations/kexue/11285/", "published_at": "2025-09-14T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18b24c594", "title": "Rethinking the Relationship between Learning Rate and Batch Size (II) - Mean Field", "url": "https://main-horse.github.io/translations/kexue/11280/", "published_at": "2025-09-09T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f17fe03f11", "title": "H-Net - Scaling Laws (Byte)", "url": "https://main-horse.github.io/hnet/scaling-laws-byte/", "published_at": "2025-09-08T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f17ecb4d9d", "title": "H-Net - Parameterization (S≥1)", "url": "https://main-horse.github.io/hnet/param-s1/", "published_at": "2025-09-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f17d5095a0", "title": "H-Net - Intuitions", "url": "https://main-horse.github.io/hnet/intuitions/", "published_at": "2025-09-04T16:00:00+00:00" }, { "id": "01a0879e-aebb-7043-b978-2eb602098103", "title": "H-Net - Engineering (1gpu)", "url": "https://main-horse.github.io/hnet/eng-1gpu/", "published_at": "2025-09-02T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18b9dad7e", "title": "Why is Adam's Update RMS 0.2?", "url": "https://main-horse.github.io/translations/kexue/11267/", "published_at": "2025-09-01T16:00:00+00:00" }, { "id": "01a0879e-aebb-7043-b978-2eb600f16c52", "title": "H-Net - Parameterization (Baseline)", "url": "https://main-horse.github.io/hnet/param-s0/", "published_at": "2025-08-31T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18c083e54", "title": "Rethinking the Relationship Between Learning Rate and Batch Size (Part 1) - Current Status", "url": "https://main-horse.github.io/translations/kexue/11260/", "published_at": "2025-08-31T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18c6101c0", "title": "4. Muon + Spectral Sphere", "url": "https://main-horse.github.io/translations/kexue/11241/", "published_at": "2025-08-20T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18d2d281f", "title": "3. Muon + Stiefel", "url": "https://main-horse.github.io/translations/kexue/11221/", "published_at": "2025-08-07T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18d4d9dc7", "title": "2. Muon + Orthogonal", "url": "https://main-horse.github.io/translations/kexue/11215/", "published_at": "2025-08-05T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18da75cfe", "title": "1. SGD + Hypersphere", "url": "https://main-horse.github.io/translations/kexue/11196/", "published_at": "2025-07-31T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18def8cb6", "title": "H-Net - Inference", "url": "https://main-horse.github.io/hnet/inf/", "published_at": "2025-07-13T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18e060f61", "title": "Advancing Muon Further on its Scale-up Journey", "url": "https://main-horse.github.io/translations/kexue/11126/", "published_at": "2025-07-11T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18eb8742e", "title": "Newton-Schulz Iteration for the msign Operator (Part II)", "url": "https://main-horse.github.io/translations/kexue/10996/", "published_at": "2025-06-04T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18fa0ea6d", "title": "A minor point about precision", "url": "https://main-horse.github.io/posts/minor-points-on-precision/", "published_at": "2025-05-25T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18fd7d1ca", "title": "Newton-Schulz Iteration for the msign Operator (Part 1)", "url": "https://main-horse.github.io/translations/kexue/10922/", "published_at": "2025-05-10T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1904d0d88", "title": "19. The Second Type of Rotary Positional Embedding", "url": "https://main-horse.github.io/translations/transformer-upgrade/10862/", "published_at": "2025-04-17T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1914b0d43", "title": "Things I learned digging into 5090 perf", "url": "https://main-horse.github.io/posts/5090-mm-perf/", "published_at": "2025-04-11T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1916a64d4", "title": "Higher-Order muP - Simpler Yet More Profound Spectral Condition Scaling", "url": "https://main-horse.github.io/translations/kexue/10795/", "published_at": "2025-03-23T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1919b4db4", "title": "Hyperparameter Scaling Laws Across Model Scales", "url": "https://main-horse.github.io/translations/kexue/10770/", "published_at": "2025-03-12T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1921b7b7e", "title": "Why Did We Choose to Try Muon?", "url": "https://main-horse.github.io/translations/kexue/10739/", "published_at": "2025-02-26T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1927e727a", "title": "Various approaches to parallelizing Muon", "url": "https://main-horse.github.io/posts/parallelizing-muon/", "published_at": "2025-02-22T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f192bba0a4", "title": "Why is the default norm length for gradient clipping 1?", "url": "https://main-horse.github.io/translations/kexue/10657/", "published_at": "2025-01-01T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f193538ba1", "title": "Reflections on Novel Weight Decay from Spectral Norm Gradient", "url": "https://main-horse.github.io/translations/kexue/10648/", "published_at": "2024-12-24T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f19403d07a", "title": "Testing the 4090 48GB", "url": "https://main-horse.github.io/posts/4090-48gb/", "published_at": "2024-12-24T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f194091886", "title": "Visualizing 6D Mesh Parallelism", "url": "https://main-horse.github.io/posts/visualizing-6d/", "published_at": "2024-12-14T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f194f57579", "title": "A Fundamental Leap from Vectors to Matrices", "url": "https://main-horse.github.io/translations/kexue/10592/", "published_at": "2024-12-09T16:00:00+00:00" }, { "id": "01a0879e-aebb-7043-b978-2eb601aa3b97", "title": "1 - Initialization and Generation of ncclUniqueId", "url": "https://main-horse.github.io/translations/nccl/initialization_and_generation_of_nccluniqueid/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f17cf24ffa", "title": "2 - Establishment of Bootstrap Network Connection", "url": "https://main-horse.github.io/translations/nccl/establishment_of_bootstrap_network_connection/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f17e232ca2", "title": "3 - Intra-machine Topology Analysis", "url": "https://main-horse.github.io/translations/nccl/intra-machine_topology_analysis/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f17f6b19c3", "title": "4 - Graph Building Process", "url": "https://main-horse.github.io/translations/nccl/graph_building_process/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f17fbf87b8", "title": "5 - Path Calculation", "url": "https://main-horse.github.io/translations/nccl/path_calculation/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1808e06e2", "title": "6 - Channel Search", "url": "https://main-horse.github.io/translations/nccl/channel_search/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f180bd6b4a", "title": "7 - Inter-machine Channel Connection", "url": "https://main-horse.github.io/translations/nccl/inter-machine_channel_connection/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18109f873", "title": "8 - Establishment of Data Communication Link Transport", "url": "https://main-horse.github.io/translations/nccl/establishment_of_data_communication_link_transport/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f181da21eb", "title": "9 - Single-machine ncclSend and ncclRecv Process", "url": "https://main-horse.github.io/translations/nccl/single-machine_ncclsend_and_ncclrecv_process/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1825a5765", "title": "10 - Inter-machine ncclSend and ncclRecv Process", "url": "https://main-horse.github.io/translations/nccl/inter-machine_ncclsend_and_ncclrecv_process/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1831c22ac", "title": "11 - Ring Allreduce", "url": "https://main-horse.github.io/translations/nccl/ring_allreduce/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18358bfb0", "title": "12 - Double Binary Tree", "url": "https://main-horse.github.io/translations/nccl/double_binary_tree/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1840aa470", "title": "13 - IB SHARP", "url": "https://main-horse.github.io/translations/nccl/ib_sharp/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f18472f730", "title": "14 - NVLink SHARP", "url": "https://main-horse.github.io/translations/nccl/nvlink_sharp/", "published_at": "2024-12-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1953401dc", "title": "Why reduction precision matters", "url": "https://main-horse.github.io/posts/reduction-precision/", "published_at": "2024-12-04T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f1962d070a", "title": "Understanding Adaptive Learning Rate Optimizers from the Perspective of Hessian Approximation", "url": "https://main-horse.github.io/translations/kexue/10588/", "published_at": "2024-11-28T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f196e729fd", "title": "How Adam's Epsilon Affects the Learning Rate Scaling Law?", "url": "https://main-horse.github.io/translations/kexue/10563/", "published_at": "2024-11-17T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f197002558", "title": "How Should the Learning Rate Change When Batch Size Increases?", "url": "https://main-horse.github.io/translations/kexue/10542/", "published_at": "2024-11-13T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f197529d87", "title": "Adding a Linear Transformation to the Codebook", "url": "https://main-horse.github.io/translations/kexue/10519/", "published_at": "2024-11-05T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f197592da5", "title": "SVD", "url": "https://main-horse.github.io/translations/kexue/10407/", "published_at": "2024-09-30T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f197eb5e90", "title": "18. Principle for RoPE's Base Selection", "url": "https://main-horse.github.io/translations/transformer-upgrade/10122/", "published_at": "2024-05-28T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f198dba379", "title": "17. Simple Thoughts on Multimodal Positional Encoding", "url": "https://main-horse.github.io/translations/transformer-upgrade/10040/", "published_at": "2024-03-28T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f199b392a7", "title": "16. A 'Review' of Length Extrapolation Techniques", "url": "https://main-horse.github.io/translations/transformer-upgrade/9948/", "published_at": "2024-01-25T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f19a22c688", "title": "15. Key Normalization Boosts Length Extrapolation", "url": "https://main-horse.github.io/translations/transformer-upgrade/9859/", "published_at": "2023-11-19T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f19af984fc", "title": "VQing the Key Makes Transformer Complexity Linear", "url": "https://main-horse.github.io/translations/kexue/9844/", "published_at": "2023-11-08T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f19b519d3b", "title": "'Rounding' Surpasses VQ-VAE", "url": "https://main-horse.github.io/translations/kexue/9826/", "published_at": "2023-10-30T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f19bcad335", "title": "14. When HWFA Meets ReRoPE", "url": "https://main-horse.github.io/translations/transformer-upgrade/9731/", "published_at": "2023-08-23T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f19c88c556", "title": "13. Inverse Use of Leaky ReRoPE", "url": "https://main-horse.github.io/translations/transformer-upgrade/9728/", "published_at": "2023-08-13T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f19cc24ad1", "title": "12. Infinitely Extrapolatable ReRoPE?", "url": "https://main-horse.github.io/translations/transformer-upgrade/9708/", "published_at": "2023-08-06T16:00:00+00:00" }, { "id": "01a0879e-aebc-7209-be67-74f19d076360", "title": "11. Carrying the Base-β Position to the End", "url": "https://main-horse.github.io/translations/transformer-upgrade/9706/", "published_at": "2023-07-30T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da476f0e5d", "title": "10. RoPE is a β-Based Encoding", "url": "https://main-horse.github.io/translations/transformer-upgrade/9675/", "published_at": "2023-07-05T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4854f63c", "title": "Exploring the Path to Minimums", "url": "https://main-horse.github.io/translations/kexue/9660/", "published_at": "2023-06-15T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da48665f13", "title": "9. A New Approach to Global Length Extrapolation", "url": "https://main-horse.github.io/translations/transformer-upgrade/9603/", "published_at": "2023-05-11T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da48a9d714", "title": "8. Length Extrapolation and Positional Robustness", "url": "https://main-horse.github.io/translations/transformer-upgrade/9444/", "published_at": "2023-01-30T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da498fe75b", "title": "7. Length Extrapolation and Local Attention", "url": "https://main-horse.github.io/translations/transformer-upgrade/9431/", "published_at": "2023-01-11T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da49dab032", "title": "6. Completeness Analysis of Rotational Positional Encoding", "url": "https://main-horse.github.io/translations/transformer-upgrade/9403/", "published_at": "2022-12-27T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4a0fdada", "title": "What's so Difficult About Training a 1000-Layer Transformer?", "url": "https://main-horse.github.io/translations/kexue/8978/", "published_at": "2022-03-08T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4a2a395a", "title": "Thoughts on Dimension Averaging Strategies for Non-Square Matrices in Initialization Methods", "url": "https://main-horse.github.io/translations/kexue/8725/", "published_at": "2021-10-17T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4a772aeb", "title": "A Brief Discussion on Initialization, Parameterization, and Normalization of Transformers", "url": "https://main-horse.github.io/translations/kexue/8620/", "published_at": "2021-08-16T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4a799629", "title": "5. Linear Attention as Infinite Dimension", "url": "https://main-horse.github.io/translations/transformer-upgrade/8601/", "published_at": "2021-08-05T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4b49a58c", "title": "4. Rotary Position Embedding for 2D Positions", "url": "https://main-horse.github.io/translations/transformer-upgrade/8397/", "published_at": "2021-05-09T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4b63a4b7", "title": "3. From Performer to Linear Attention", "url": "https://main-horse.github.io/translations/transformer-upgrade/8338/", "published_at": "2021-04-21T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4baeae74", "title": "2. Rotary Position Embedding with Diverse Strengths", "url": "https://main-horse.github.io/translations/transformer-upgrade/8265/", "published_at": "2021-03-22T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4be2085e", "title": "1. Tracing the Origins of Sinusoidal Position Encoding", "url": "https://main-horse.github.io/translations/transformer-upgrade/8231/", "published_at": "2021-03-07T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4c6a47cc", "title": "Understanding Model Parameter Initialization Strategies from a Geometric Perspective", "url": "https://main-horse.github.io/translations/kexue/7180/", "published_at": "2020-01-15T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4d0e27d1", "title": "Distribution of the Angle Between Two Random Vectors in $n$-Dimensional Space", "url": "https://main-horse.github.io/translations/kexue/7076/", "published_at": "2019-11-12T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4e01c1b7", "title": "What Does BN Actually Do? An Analysis From First Principles", "url": "https://main-horse.github.io/translations/kexue/6992/", "published_at": "2019-10-10T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4ea4f349", "title": "Vector Quantized AutoEncoder", "url": "https://main-horse.github.io/translations/kexue/6760/", "published_at": "2019-06-23T16:00:00+00:00" }, { "id": "01a0879e-aebd-702f-a427-92da4f72335b", "title": "Generalization and Generative Models", "url": "https://main-horse.github.io/translations/kexue/6051/", "published_at": "2018-10-06T16:00:00+00:00" } ] posts Claim your blog
Back to main-horse.github.io
Blog · corpus.blog/blogs/main-horse.github.io/posts

main-horse.github.io

main-horse.github.io

2025

Muon Optimizer Guide - Quick Start and Key Detailsoriginal ↗
18 Nov 2025
Asymptotic Estimation of AdamW's Weight RMS (Part 2)original ↗
16 Nov 2025
Asymptotic Estimate for the Maximum of n Normal Random Variablesoriginal ↗
5 Nov 2025
5. Dual Gradient Descentoriginal ↗
2 Nov 2025
Low-Precision Attention May Suffer from Biased Rounding Errorsoriginal ↗
26 Oct 2025
1. Three Characteristics of a Good Modeloriginal ↗
20 Oct 2025
Fast Estimation of the Spectral Norm of Random Matricesoriginal ↗
11 Oct 2025
A Very Concise VQ Training Schemeoriginal ↗
7 Oct 2025
Why Add Short Conv to Linear Attention?original ↗
4 Oct 2025
Asymptotic Estimation of AdamW's Weight RMSoriginal ↗
30 Sept 2025
Rethinking Learning Rate and Batch Size (IV) - EMAoriginal ↗
21 Sept 2025
Rethinking Learning Rate and Batch Size (III) - Muonoriginal ↗
14 Sept 2025
Rethinking the Relationship between Learning Rate and Batch Size (II) - Mean Fieldoriginal ↗
9 Sept 2025
H-Net - Scaling Laws (Byte)original ↗
8 Sept 2025
H-Net - Parameterization (S≥1)original ↗
6 Sept 2025
H-Net - Intuitionsoriginal ↗
4 Sept 2025
H-Net - Engineering (1gpu)original ↗
2 Sept 2025
Why is Adam's Update RMS 0.2?original ↗
1 Sept 2025
H-Net - Parameterization (Baseline)original ↗
31 Aug 2025
Rethinking the Relationship Between Learning Rate and Batch Size (Part 1) - Current Statusoriginal ↗
31 Aug 2025
4. Muon + Spectral Sphereoriginal ↗
20 Aug 2025
3. Muon + Stiefeloriginal ↗
7 Aug 2025
2. Muon + Orthogonaloriginal ↗
5 Aug 2025
1. SGD + Hypersphereoriginal ↗
31 Jul 2025
H-Net - Inferenceoriginal ↗
13 Jul 2025
Advancing Muon Further on its Scale-up Journeyoriginal ↗
11 Jul 2025
Newton-Schulz Iteration for the msign Operator (Part II)original ↗
4 Jun 2025
A minor point about precisionoriginal ↗
25 May 2025
Newton-Schulz Iteration for the msign Operator (Part 1)original ↗
10 May 2025
19. The Second Type of Rotary Positional Embeddingoriginal ↗
17 Apr 2025
Things I learned digging into 5090 perforiginal ↗
11 Apr 2025
Higher-Order muP - Simpler Yet More Profound Spectral Condition Scalingoriginal ↗
23 Mar 2025
Hyperparameter Scaling Laws Across Model Scalesoriginal ↗
12 Mar 2025
Why Did We Choose to Try Muon?original ↗
26 Feb 2025
Various approaches to parallelizing Muonoriginal ↗
22 Feb 2025
Why is the default norm length for gradient clipping 1?original ↗
1 Jan 2025

2024

Reflections on Novel Weight Decay from Spectral Norm Gradientoriginal ↗
24 Dec 2024
Testing the 4090 48GBoriginal ↗
24 Dec 2024
Visualizing 6D Mesh Parallelismoriginal ↗
14 Dec 2024
A Fundamental Leap from Vectors to Matricesoriginal ↗
9 Dec 2024
1 - Initialization and Generation of ncclUniqueIdoriginal ↗
6 Dec 2024
2 - Establishment of Bootstrap Network Connectionoriginal ↗
6 Dec 2024
3 - Intra-machine Topology Analysisoriginal ↗
6 Dec 2024
4 - Graph Building Processoriginal ↗
6 Dec 2024
5 - Path Calculationoriginal ↗
6 Dec 2024
6 - Channel Searchoriginal ↗
6 Dec 2024
7 - Inter-machine Channel Connectionoriginal ↗
6 Dec 2024
8 - Establishment of Data Communication Link Transportoriginal ↗
6 Dec 2024
9 - Single-machine ncclSend and ncclRecv Processoriginal ↗
6 Dec 2024
10 - Inter-machine ncclSend and ncclRecv Processoriginal ↗
6 Dec 2024
11 - Ring Allreduceoriginal ↗
6 Dec 2024
12 - Double Binary Treeoriginal ↗
6 Dec 2024
13 - IB SHARPoriginal ↗
6 Dec 2024
14 - NVLink SHARPoriginal ↗
6 Dec 2024
Why reduction precision mattersoriginal ↗
4 Dec 2024
Understanding Adaptive Learning Rate Optimizers from the Perspective of Hessian Approximationoriginal ↗
28 Nov 2024
How Adam's Epsilon Affects the Learning Rate Scaling Law?original ↗
17 Nov 2024
How Should the Learning Rate Change When Batch Size Increases?original ↗
13 Nov 2024
Adding a Linear Transformation to the Codebookoriginal ↗
5 Nov 2024
SVDoriginal ↗
30 Sept 2024
18. Principle for RoPE's Base Selectionoriginal ↗
28 May 2024
17. Simple Thoughts on Multimodal Positional Encodingoriginal ↗
28 Mar 2024
16. A 'Review' of Length Extrapolation Techniquesoriginal ↗
25 Jan 2024

2023

15. Key Normalization Boosts Length Extrapolationoriginal ↗
19 Nov 2023
VQing the Key Makes Transformer Complexity Linearoriginal ↗
8 Nov 2023
'Rounding' Surpasses VQ-VAEoriginal ↗
30 Oct 2023
14. When HWFA Meets ReRoPEoriginal ↗
23 Aug 2023
13. Inverse Use of Leaky ReRoPEoriginal ↗
13 Aug 2023
12. Infinitely Extrapolatable ReRoPE?original ↗
6 Aug 2023
11. Carrying the Base-β Position to the Endoriginal ↗
30 Jul 2023
10. RoPE is a β-Based Encodingoriginal ↗
5 Jul 2023
Exploring the Path to Minimumsoriginal ↗
15 Jun 2023
9. A New Approach to Global Length Extrapolationoriginal ↗
11 May 2023
8. Length Extrapolation and Positional Robustnessoriginal ↗
30 Jan 2023
7. Length Extrapolation and Local Attentionoriginal ↗
11 Jan 2023

2022

6. Completeness Analysis of Rotational Positional Encodingoriginal ↗
27 Dec 2022
What's so Difficult About Training a 1000-Layer Transformer?original ↗
8 Mar 2022

2021

Thoughts on Dimension Averaging Strategies for Non-Square Matrices in Initialization Methodsoriginal ↗
17 Oct 2021
A Brief Discussion on Initialization, Parameterization, and Normalization of Transformersoriginal ↗
16 Aug 2021
5. Linear Attention as Infinite Dimensionoriginal ↗
5 Aug 2021
4. Rotary Position Embedding for 2D Positionsoriginal ↗
9 May 2021
3. From Performer to Linear Attentionoriginal ↗
21 Apr 2021
2. Rotary Position Embedding with Diverse Strengthsoriginal ↗
22 Mar 2021
1. Tracing the Origins of Sinusoidal Position Encodingoriginal ↗
7 Mar 2021

2020

Understanding Model Parameter Initialization Strategies from a Geometric Perspectiveoriginal ↗
15 Jan 2020

2019

Distribution of the Angle Between Two Random Vectors in $n$-Dimensional Spaceoriginal ↗
12 Nov 2019
What Does BN Actually Do? An Analysis From First Principlesoriginal ↗
10 Oct 2019
Vector Quantized AutoEncoderoriginal ↗
23 Jun 2019

2018

Generalization and Generative Modelsoriginal ↗
6 Oct 2018