Handling 10K+ Concurrent Connections in Go

// DON'T use in production for high concurrency
listener, _ := net.Listen("tcp", ":8080")
for {
    conn, _ := listener.Accept()
    // Each connection spawns an OS thread (expensive!)
    go handleConnection(conn) // But this is a goroutine, not a thread!
}

// Simplified representation of netpoller internals
type pollDesc struct {
    fd    uintptr       // file descriptor
    rg    *g            // goroutine waiting for read
    wg    *g            // goroutine waiting for write
    // ... other fields
}

type pollCache struct {
    lock  mutex
    first *pollDesc
    // ... pool of pre-allocated pollDesc
}

// Conceptual flow in runtime/poll
func (pd *pollDesc) waitRead(deadline int64) error {
    if pd.rg != 0 {
        return errors.New("read already in progress")
    }

    gp := getg()
    pd.rg.store(gp)

    // Register with netpoller
    netpollarm(pd, 'r')

    // Park the goroutine - will resume when data arrives
    gopark(sysmon, unsafe.Pointer(pd), waitReasonIOWait)

    return nil
}

// How Go's epoll wrapper works (simplified)
func epollWait(ep uintptr, events []epollevent, timeout int32) int32 {
    return syscall.EpollWait(ep, events, timeout)
}

// Runtime makes a system call to epoll_wait
// This blocks until events arrive or timeout occurs
// Default timeout: 0 (no timeout)
n, _ := syscall.EpollWait(epfd, events[:], -1)

// Process all events in one go
for i := 0; i < n; i++ {
    // Wake up the appropriate goroutine
    // based on events[i].Fd and events[i].Events
}

Initial goroutine stack: 2,048 bytes

Breakdown (approximate):
- Stack frame pointers:      ~64 bytes
- Local variables space:     ~1,900 bytes
- Guard page/metadata:       ~84 bytes

// Go's stack growth mechanism
func growStack(sp uintptr, requiredSize uintptr) {
    // 1. Allocate new, larger stack (typically 2x)
    // 2. Copy existing frames to new stack
    // 3. Update all frame pointers
    // 4. Resume execution

    // This happens transparently at runtime
}

Goroutine layer (M):          G1  G2  G3  ... G10000
                              |   |   |
Scheduler layer (M:N):    [P0 queue][P1 queue][P2 queue]
                              |         |         |
OS thread layer (N):          M1        M2        M3

Each P (processor) has:
- One OS thread (M)
- Local run queue (limited size)
- Global fallback queue

// Example: 10 goroutines, 1 OS thread
// Each blocked on network I/O

for i := 0; i < 10; i++ {
    go func() {
        resp, _ := http.Get("https://api.example.com/data")
        process(resp)
    }()
}

// The single OS thread:
// Time 1ms: G1 blocked on DNS → switch to G2
// Time 2ms: G2 blocked on TLS → switch to G3
// Time 3ms: G1's DNS completes → queue G1 for execution
// etc...

package main

import (
    "bufio"
    "log"
    "net"
)

func main() {
    listener, err := net.Listen("tcp", ":8080")
    if err != nil {
        log.Fatal(err)
    }
    defer listener.Close()

    log.Printf("Server listening on %s", listener.Addr())

    for {
        // Accept returns immediately when a connection is available
        conn, err := listener.Accept()
        if err != nil {
            log.Printf("Accept error: %v", err)
            continue
        }

        // Spawn a goroutine for this connection
        // This is cheap: ~2KB + buffers
        go handleConnection(conn)
    }
}

func handleConnection(conn net.Conn) {
    defer conn.Close()

    // Use buffered I/O
    reader := bufio.NewReader(conn)
    writer := bufio.NewWriter(conn)

    for {
        // Read from client
        line, err := reader.ReadString('\n')
        if err != nil {
            return
        }

        // Process request
        response := processRequest(line)

        // Write response
        writer.WriteString(response)
        writer.Flush()
    }
}

func processRequest(req string) string {
    // Business logic here
    return "OK\n"
}

// net.Listen returns a *net.TCPListener
listener, _ := net.Listen("tcp", ":8080")

// Internally:
// - Creates a socket
// - Calls bind(2) to bind to address
// - Calls listen(2) to mark as listening
// - Registers socket with netpoller

// Accept() is registered with netpoller:
// - If no connections pending, park current goroutine
// - When connection arrives, netpoller wakes goroutine
// - Accept returns immediately

conn, _ := listener.Accept() // Efficient!

Goroutine overhead:
  - Stack (initial):                      2 KB
  - Stack (typical workload):             4-8 KB
  - Goroutine structure (g):              ~376 bytes

Connection-related:
  - net.Conn interface + implementation:  ~200 bytes
  - net.TCPConn struct:                   ~88 bytes

Buffering (default):
  - bufio.Reader (4KB buffer):            ~4 KB
  - bufio.Writer (4KB buffer):            ~4 KB

Total per connection: ~14-20 KB (depending on stack growth)

const (
    perConnectionBytes = 20 * 1024 // 20 KB conservative estimate
    availableMemoryMB  = 1024      // 1 GB available
    availableMemoryB   = availableMemoryMB * 1024 * 1024
)

maxConnections := availableMemoryB / perConnectionBytes
// Result: ~52,000 connections on 1GB dedicated memory

// Real world: leave headroom for buffers, message data, etc.
safeMaxConnections := maxConnections / 2 // ~26,000 connections

1,000 connections:
  - Goroutines: ~2 MB
  - Buffers: ~8 MB
  - Connection state: ~200 KB
  - Total: ~11 MB

10,000 connections:
  - Goroutines: ~20 MB
  - Buffers: ~80 MB
  - Connection state: ~2 MB
  - Total: ~102 MB

100,000 connections:
  - Goroutines: ~200 MB
  - Buffers: ~800 MB
  - Connection state: ~20 MB
  - Total: ~1 GB

// Reduce memory by sharing buffers across connections
var bufferPool = sync.Pool{
    New: func() interface{} {
        return make([]byte, 4096)
    },
}

func handleConnection(conn net.Conn) {
    // Borrow from pool instead of allocating
    buf := bufferPool.Get().([]byte)
    defer bufferPool.Put(buf)

    for {
        n, err := conn.Read(buf)
        if err != nil {
            return
        }

        process(buf[:n])
    }
}

# Check current limits
ulimit -n              # soft limit
ulimit -Hn             # hard limit (root can increase)

# Output on typical system:
# 1024
# 65536

# Increase for current shell
ulimit -n 100000

# Verify
ulimit -n
# Output: 100000

# Edit /etc/security/limits.conf
# Add:
* soft nofile 1000000
* hard nofile 1000000
root soft nofile 1000000
root hard nofile 1000000

# Apply changes (logout/login required)
# OR use systemctl for services

package main

import (
    "log"
    "syscall"
)

func setFileDescriptorLimit() error {
    var limit syscall.Rlimit

    // Get current limits
    if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
        return err
    }

    log.Printf("Current FD limit - Soft: %v, Hard: %v",
        limit.Cur, limit.Max)

    // Increase to hard limit
    limit.Cur = limit.Max

    if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
        return err
    }

    log.Printf("New FD limit - Soft: %v, Hard: %v",
        limit.Cur, limit.Max)

    return nil
}

func main() {
    if err := setFileDescriptorLimit(); err != nil {
        log.Fatalf("Failed to set FD limit: %v", err)
    }

    // Now start server
    startServer()
}

# Check port usage on Linux
ss -tan | grep ESTABLISHED | wc -l

# View port range
cat /proc/sys/net/ipv4/ip_local_port_range
# Output: 32768  60999 (roughly 28k available ports)

# Increase ephemeral port range
sudo sysctl -w net.ipv4.ip_local_port_range="1024 65535"

# Make permanent in /etc/sysctl.conf
# net.ipv4.ip_local_port_range = 1024 65535

// With single listener on 4-core system:
// - Only one OS thread handles accepts
// - Bottleneck at single listener
// - ~60% CPU utilization at best

listener, _ := net.Listen("tcp", ":8080")
// ...
go handleConnections(listener)

import "golang.org/x/sys/unix"

func listenReusePort(addr string, numListeners int) ([]net.Listener, error) {
    listeners := make([]net.Listener, numListeners)

    for i := 0; i < numListeners; i++ {
        // Create raw socket
        fd, err := unix.Socket(unix.AF_INET, unix.SOCK_STREAM, 0)
        if err != nil {
            return nil, err
        }

        // Enable SO_REUSEPORT
        if err := unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
            unix.Close(fd)
            return nil, err
        }

        // Bind and listen...
        // (implementation continues)

        listeners[i] = listener
    }

    return listeners, nil
}

package main

import (
    "context"
    "log"
    "net"
    "sync"
)

func main() {
    numListeners := runtime.NumCPU()
    var wg sync.WaitGroup

    for i := 0; i < numListeners; i++ {
        wg.Add(1)
        go func(id int) {
            defer wg.Done()

            // Use net.Listen with SO_REUSEPORT
            // Note: Requires unix.SetsockoptInt or stdlib support
            listener, err := net.Listen("tcp", ":8080")
            if err != nil {
                log.Fatal(err)
            }
            defer listener.Close()

            log.Printf("Listener %d started", id)

            for {
                conn, err := listener.Accept()
                if err != nil {
                    return
                }
                go handleConnection(conn)
            }
        }(i)
    }

    wg.Wait()
}

func handleWithStandardBuffer(conn net.Conn) {
    defer conn.Close()

    // Each connection gets its own 4KB buffers
    reader := bufio.NewReader(conn)
    writer := bufio.NewWriter(conn)

    for {
        line, _ := reader.ReadString('\n')
        writer.WriteString(processRequest(line))
        writer.Flush()
    }
}

// Memory cost: ~8KB per connection

var (
    readerPool = sync.Pool{
        New: func() interface{} {
            return bufio.NewReaderSize(nil, 4096)
        },
    }
    writerPool = sync.Pool{
        New: func() interface{} {
            return bufio.NewWriterSize(nil, 4096)
        },
    }
)

func handleWithPooledBuffer(conn net.Conn) {
    defer conn.Close()

    reader := readerPool.Get().(*bufio.Reader)
    writer := writerPool.Get().(*bufio.Writer)

    defer readerPool.Put(reader)
    defer writerPool.Put(writer)

    // Reset readers/writers to use new connection
    reader.Reset(conn)
    writer.Reset(conn)

    for {
        line, _ := reader.ReadString('\n')
        writer.WriteString(processRequest(line))
        writer.Flush()
    }
}

// Memory cost: ~100 bytes per connection (just wrapper)
// + amortized buffer allocation from pool

const bufferSize = 4096

var bufferPool = sync.Pool{
    New: func() interface{} {
        return make([]byte, bufferSize)
    },
}

func handleWithByteBuffer(conn net.Conn) {
    defer conn.Close()

    for {
        // Get buffer from pool
        buf := bufferPool.Get().([]byte)

        n, err := conn.Read(buf)
        if err != nil {
            bufferPool.Put(buf)
            return
        }

        // Process data
        process(buf[:n])

        // Return to pool
        bufferPool.Put(buf)
    }
}

Without pooling:
  - Allocations: 2 buffers × 10,000 = 20,000 allocs/sec
  - GC pause time: 15-50ms every 30 seconds
  - Per-connection memory: 8KB

With sync.Pool:
  - Allocations: 0 (after warmup)
  - GC pause time: 1-5ms
  - Per-connection memory: 100 bytes

Improvement: 80% reduction in GC pressure

type ConnectionHandler struct {
    reader *bufio.Reader
    writer *bufio.Writer
}

func NewHandler(conn net.Conn) *ConnectionHandler {
    return &ConnectionHandler{
        // Lazy: don't allocate until needed
        reader: nil,
        writer: nil,
    }
}

func (h *ConnectionHandler) getReader(conn net.Conn) *bufio.Reader {
    if h.reader == nil {
        h.reader = bufio.NewReader(conn)
    }
    return h.reader
}

func (h *ConnectionHandler) getWriter(conn net.Conn) *bufio.Writer {
    if h.writer == nil {
        h.writer = bufio.NewWriter(conn)
    }
    return h.writer
}

// Result: connections that only send (no read) save 4KB
// connections that only receive (no write) save 4KB

// Instead of buffering entire message:
func handleWithStreamingParse(conn net.Conn) {
    defer conn.Close()

    // Small fixed-size buffer
    buf := make([]byte, 256)

    for {
        n, err := conn.Read(buf)
        if err != nil {
            return
        }

        // Process message by message
        // without intermediate buffering
        processStream(buf[:n])
    }
}

type Connection struct {
    Conn     net.Conn
    Protocol ProtocolParser
    State    ConnectionState
}

var connPool = sync.Pool{
    New: func() interface{} {
        return &Connection{}
    },
}

func handleWithConnPooling(conn net.Conn) {
    pooledConn := connPool.Get().(*Connection)
    pooledConn.Conn = conn
    defer func() {
        pooledConn.Conn = nil
        pooledConn.Protocol.Reset()
        connPool.Put(pooledConn)
    }()

    // Handle connection
}

package main

import (
    "fmt"
    "runtime"
)

func main() {
    // Default: GOMAXPROCS = number of CPU cores
    maxProcs := runtime.NumCPU()
    fmt.Printf("NumCPU: %d\n", maxProcs)

    // You can override:
    runtime.GOMAXPROCS(maxProcs)
}

package main

import (
    "log"
    "runtime"
)

func main() {
    numCPU := runtime.NumCPU()

    // Option 1: Match CPU count (default)
    // Good for CPU-bound work with network I/O
    runtime.GOMAXPROCS(numCPU)

    // Option 2: Increase for I/O-heavy workloads
    // Each goroutine can block on I/O
    // More OS threads = better latency
    desiredThreads := numCPU * 2
    runtime.GOMAXPROCS(desiredThreads)

    // Option 3: Set via environment (simpler)
    // GOMAXPROCS=16 go run server.go
}

# Increase TCP backlog
sudo sysctl -w net.ipv4.tcp_max_syn_backlog=65535
sudo sysctl -w net.core.somaxconn=65535

# Improve connection timeout behavior
sudo sysctl -w net.ipv4.tcp_fin_timeout=30
sudo sysctl -w net.ipv4.tcp_tw_reuse=1

# Increase network buffer sizes
sudo sysctl -w net.core.rmem_max=134217728
sudo sysctl -w net.core.wmem_max=134217728
sudo sysctl -w net.ipv4.tcp_rmem="4096 87380 134217728"
sudo sysctl -w net.ipv4.tcp_wmem="4096 65536 134217728"

package main

import (
    "context"
    "log"
    "net"
    "sync"
    "syscall"
    "os"
    "os/signal"
)

type Server struct {
    listener net.Listener
    mu       sync.RWMutex
    conns    map[net.Conn]bool
    ctx      context.Context
    cancel   context.CancelFunc
}

func NewServer(addr string) (*Server, error) {
    listener, err := net.Listen("tcp", addr)
    if err != nil {
        return nil, err
    }

    ctx, cancel := context.WithCancel(context.Background())

    return &Server{
        listener: listener,
        conns:    make(map[net.Conn]bool),
        ctx:      ctx,
        cancel:   cancel,
    }, nil
}

func (s *Server) Start() error {
    go s.acceptLoop()

    // Wait for shutdown signal
    sigChan := make(chan os.Signal, 1)
    signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
    <-sigChan

    return s.Shutdown()
}

func (s *Server) acceptLoop() {
    for {
        conn, err := s.listener.Accept()

        select {
        case <-s.ctx.Done():
            return
        default:
        }

        if err != nil {
            log.Printf("Accept error: %v", err)
            continue
        }

        // Track connection
        s.mu.Lock()
        s.conns[conn] = true
        s.mu.Unlock()

        go s.handleConnection(conn)
    }
}

func (s *Server) handleConnection(conn net.Conn) {
    defer func() {
        conn.Close()
        s.mu.Lock()
        delete(s.conns, conn)
        s.mu.Unlock()
    }()

    // Handle I/O with context awareness
    for {
        select {
        case <-s.ctx.Done():
            log.Println("Server shutting down, closing connection")
            return
        default:
        }

        // Read/write with timeout
        // ...
    }
}

func (s *Server) Shutdown() error {
    log.Println("Starting graceful shutdown...")

    s.cancel()
    s.listener.Close()

    // Wait for all connections to close
    s.mu.RLock()
    numConns := len(s.conns)
    s.mu.RUnlock()

    for numConns > 0 {
        log.Printf("Waiting for %d connections to close...", numConns)
        time.Sleep(100 * time.Millisecond)

        s.mu.RLock()
        numConns = len(s.conns)
        s.mu.RUnlock()
    }

    log.Println("Graceful shutdown complete")
    return nil
}

package main

import (
    "log"
    "net"
    "sync"
    "sync/atomic"
    "time"
)

type ConnectionMonitor struct {
    activeCount   int64  // atomic
    totalCount    int64  // atomic
    peakCount     int64  // atomic
    createdTime   time.Time
    mu            sync.RWMutex
    connections   map[string]*connInfo
}

type connInfo struct {
    createdAt  time.Time
    lastActive time.Time
    bytesRead  int64
    bytesWrite int64
}

func (m *ConnectionMonitor) OnConnect(addr string) {
    atomic.AddInt64(&m.activeCount, 1)
    atomic.AddInt64(&m.totalCount, 1)

    m.mu.Lock()
    m.connections[addr] = &connInfo{
        createdAt:  time.Now(),
        lastActive: time.Now(),
    }
    m.mu.Unlock()

    // Update peak
    for {
        current := atomic.LoadInt64(&m.activeCount)
        peak := atomic.LoadInt64(&m.peakCount)
        if current <= peak || atomic.CompareAndSwapInt64(&m.peakCount, peak, current) {
            break
        }
    }
}

func (m *ConnectionMonitor) OnDisconnect(addr string) {
    atomic.AddInt64(&m.activeCount, -1)

    m.mu.Lock()
    delete(m.connections, addr)
    m.mu.Unlock()
}

func (m *ConnectionMonitor) Stats() map[string]int64 {
    return map[string]int64{
        "active": atomic.LoadInt64(&m.activeCount),
        "total":  atomic.LoadInt64(&m.totalCount),
        "peak":   atomic.LoadInt64(&m.peakCount),
    }
}

func (m *ConnectionMonitor) PrintStats() {
    stats := m.Stats()
    log.Printf("Connections - Active: %d, Total: %d, Peak: %d",
        stats["active"], stats["total"], stats["peak"])
}

// Usage
var monitor = &ConnectionMonitor{
    connections: make(map[string]*connInfo),
    createdTime: time.Now(),
}

func handleConnection(conn net.Conn) {
    monitor.OnConnect(conn.RemoteAddr().String())
    defer monitor.OnDisconnect(conn.RemoteAddr().String())

    // Handle connection...
}

func init() {
    go func() {
        ticker := time.NewTicker(10 * time.Second)
        defer ticker.Stop()

        for range ticker.C {
            monitor.PrintStats()
        }
    }()
}

package main

import (
    "log"
    "net"
    "runtime"
    "sync"
    "sync/atomic"
    "testing"
    "time"
)

func BenchmarkConnections(b *testing.B) {
    concurrencyLevels := []int{1000, 10000, 50000, 100000}

    for _, level := range concurrencyLevels {
        b.Run(fmt.Sprintf("connections-%d", level), func(b *testing.B) {
            benchmarkConnectionLevel(b, level)
        })
    }
}

func benchmarkConnectionLevel(b *testing.B, numConns int) {
    // Start server
    listener, _ := net.Listen("tcp", ":0") // random port
    defer listener.Close()

    var (
        wg             sync.WaitGroup
        activeConns    int64
        processedMsgs  int64
    )

    // Server goroutine
    go func() {
        for {
            conn, _ := listener.Accept()
            if conn == nil {
                return
            }

            atomic.AddInt64(&activeConns, 1)

            wg.Add(1)
            go func(c net.Conn) {
                defer wg.Done()
                defer c.Close()
                defer atomic.AddInt64(&activeConns, -1)

                buf := make([]byte, 1024)
                for {
                    n, err := c.Read(buf)
                    if err != nil {
                        return
                    }
                    atomic.AddInt64(&processedMsgs, 1)
                    c.Write(buf[:n])
                }
            }(conn)
        }
    }()

    time.Sleep(100 * time.Millisecond) // warmup

    var m runtime.MemStats
    runtime.ReadMemStats(&m)
    startMem := m.Alloc

    b.ResetTimer()

    // Connect clients
    var clientWg sync.WaitGroup
    for i := 0; i < numConns; i++ {
        clientWg.Add(1)
        go func() {
            defer clientWg.Done()
            conn, _ := net.Dial("tcp", listener.Addr().String())
            defer conn.Close()

            for j := 0; j < 10; j++ {
                conn.Write([]byte("test"))
                buf := make([]byte, 4)
                conn.Read(buf)
            }
        }()
    }

    clientWg.Wait()
    b.StopTimer()

    // Metrics
    runtime.ReadMemStats(&m)
    memUsed := (m.Alloc - startMem) / 1024 / 1024

    log.Printf("Connections: %d, Memory: %dMB, Msgs/sec: %d, Active: %d",
        numConns, memUsed, atomic.LoadInt64(&processedMsgs),
        atomic.LoadInt64(&activeConns))
}

connections-1000:    15 MB memory,  ~95% efficiency
connections-10000:   120 MB memory, ~92% efficiency
connections-50000:   600 MB memory, ~88% efficiency
connections-100000:  1200 MB memory, ~80% efficiency

Per-connection memory:
1K:    15 KB  (baseline overhead)
10K:   12 KB  (baseline amortized)
100K:  12 KB  (baseline amortized)

import "github.com/panjf2000/gnet"

func main() {
    var eventHandler EventHandler

    engine, err := gnet.New("tcp://:8080", eventHandler)
    // ...
}

// EventHandler implements:
// OnOpen(gnet.Conn) error
// OnClose(gnet.Conn) error
// OnTraffic(gnet.Conn) error

import "github.com/tidwall/evio"

func main() {
    evio.Serve(events, "tcp://:8080")
}

type events struct{}

func (e *events) OnOpen(conn evio.Conn) (out []byte, opts evio.Options, rerr error) {
    return
}

func (e *events) OnData(conn evio.Conn, in []byte) (out []byte, opts evio.Options, rerr error) {
    // Process data
    return
}