Go Distributed System

Go

High-performance distributed job processing system built with Go

Tech Stack

Go 1.22
gRPC
Apache Kafka
Redis
PostgreSQL
Kubernetes
Docker

Features

✓
gRPC API gateway
✓
Kafka message queue
✓
Concurrent worker pool
✓
Redis caching layer
✓
Kubernetes orchestration
✓
PostgreSQL storage

Code Sample

worker.go
package worker

import (
    "context"
    "sync"
)

type WorkerPool struct {
    jobs    chan Job
    results chan Result
    wg      sync.WaitGroup
}

func (wp *WorkerPool) Start(ctx context.Context, n int) {
    for i := 0; i < n; i++ {
        wp.wg.Add(1)
        go wp.worker(ctx)
    }
}

func (wp *WorkerPool) worker(ctx context.Context) {
    defer wp.wg.Done()
    for job := range wp.jobs {
        select {
        case <-ctx.Done():
            return
        default:
            result := job.Execute(ctx)
            wp.results <- result
        }
    }
}
View Source Docker Hub Live Demo