ashwanthgoli / ziggurat

Ziggurat Golang

Stream Processing made easy

Install the ziggurat CLI

go get -v -u github.com/gojekfarm/ziggurat/cmd/...
go install github.com/gojekfarm/ziggurat/cmd/...                                                                                                                                                     

How to use

create a new app using the new command

ziggurat new <app_name>

Main file

package main

import (
	"context"
	"fmt"
	"github.com/gojekfarm/ziggurat"
	"github.com/gojekfarm/ziggurat/kafka"
	"github.com/gojekfarm/ziggurat/logger"
	"github.com/gojekfarm/ziggurat/mw"
	"github.com/gojekfarm/ziggurat/router"
)

func main() {
	jsonLogger := logger.NewJSONLogger("info")
	ctx := context.Background()

	kafkaStreams := &kafka.Streams{
		StreamConfig: kafka.StreamConfig{
			{
				BootstrapServers: "localhost:9092",
				OriginTopics:     "plain-text-log",
				ConsumerGroupID:  "plain_text_consumer",
				ConsumerCount:    1,
				RouteGroup:       "plain-text-log",
			},
			{
				BootstrapServers: "localhost:9092",
				OriginTopics:     "json-log",
				ConsumerGroupID:  "json_consumer",
				ConsumerCount:    1,
				RouteGroup:       "json-log",
			},
		},
		Logger: jsonLogger,
	}
	r := router.New()

	r.HandleFunc("plain-text-log", func(event ziggurat.Event, ctx context.Context) error {
		return nil
	})

	r.HandleFunc("json-log", func(event ziggurat.Event, ctx context.Context) error {
		return ziggurat.ErrProcessingFailed{}
	})

	handler := &mw.ProcessingStatusLogger{Logger: jsonLogger, Handler: r}

	zig := &ziggurat.Ziggurat{Logger: jsonLogger}

	if runErr := zig.Run(ctx, kafkaStreams, handler); runErr != nil {
		fmt.Println("error running streams ", runErr)
	}

}

Concepts

• There are 3 main interfaces that your structs can implement to plug into the Ziggurat library

Streamer interface

package ziggurat

import "context"

type Streamer interface {
	Stream(ctx context.Context, handler Handler) error
}

// Any type can implement the Streamer interface to stream data from any source

Handler interface

package ziggurat

import "context"

type Handler interface {
	Handle(event Event, ctx context.Context) error
}

// The Handler interface is very similar to the http.Handler interface
// The default router shipped with Ziggurat also implements the Handler interface

Event interface

package ziggurat

type Event interface {
	Value() []byte
	Key() 	[]byte
	Headers() map[string]string
}

// Every stream must produce a series of events to be handled by the handler
// An event typically returns a byte value, headers which can contain additional metadata