Go Distributed System

Go

High-performance distributed job processing system built with Go

Tech Stack

Go 1.22
gRPC
Apache Kafka
Redis
PostgreSQL
Kubernetes
Docker

Features

gRPC API gateway
Kafka message queue
Concurrent worker pool
Redis caching layer
Kubernetes orchestration
PostgreSQL storage

Code Sample

worker.go
package worker

import (
    "context"
    "sync"
)

type WorkerPool struct {
    jobs    chan Job
    results chan Result
    wg      sync.WaitGroup
}

func (wp *WorkerPool) Start(ctx context.Context, n int) {
    for i := 0; i < n; i++ {
        wp.wg.Add(1)
        go wp.worker(ctx)
    }
}

func (wp *WorkerPool) worker(ctx context.Context) {
    defer wp.wg.Done()
    for job := range wp.jobs {
        select {
        case <-ctx.Done():
            return
        default:
            result := job.Execute(ctx)
            wp.results <- result
        }
    }
}
View Source Docker Hub Live Demo