How to syncing data between application instances? Easy!

Syncing data with 12k requests/sec messages between instances

Replication. What is it?

Replication description from the Wiki:
Replication in computing involves sharing information so as to ensure consistency between redundant resources, such as software or hardware components, to improve reliability, fault-tolerance, or accessibility.
Me after a week recover lost data from the broken server

Ok. Suppose I convinced you and you want to make your own replication in your application

First steps. Making the transaction model.

// In my opinion, protobufs is the best solution for the replication.
// Our identifier will be unix nano timestamp
syntax = "proto3";

message Transaction {
int64 timestamp = 1;
map<string, bytes> data = 2;
}
var transactionPool = sync.Pool{
New: func() interface{} {
return new(Transaction)
},
}

func GetTransaction() *Transaction {
return transactionPool.Get().(*Transaction)
}

func PutTransaction(transaction *Transaction) {
transaction.Reset()
transactionPool.Put(transaction)
}
func NewTransaction(data map[string][]byte) *Transaction {
transaction := GetTransaction()
transaction.Timestamp = time.Now().UnixNano()
transaction.Data = data
return transaction
}

Making the replication.

type Replication struct {
Transmitter chan *Transaction
Receiver chan *Transaction

Received map[string]int64

Replicas map[string]string
}
func (r *Replication) Transmit() {
for {
transaction := <-r.Transmitter

go func() {
body, err := proto.Marshal(transaction)

if err == nil {
for addr, host := range r.Replicas {
//That's url what you services will listen for transactions
_, err := http.Post(host + "/replication/receiver", "application/protobuf", bytes.NewReader(body))

if err != nil {
log.Println("Master", addr, "is down:", err)
}
}
} else {
log.Println("Replication error:", err)
}

PutTransaction(transaction)
}()
}
}
func (r *Replication) Receive() {
for {
transaction := <-r.Receiver

go func() {
for key, data := range transaction.Data {
if timestamp, ok := r.Received[key]; ok && timestamp <= transaction.Timestamp {
continue
}
// Put transaction data to you data store
err := database.Client.Put(key, data)

if err == nil {
r.Received[key] = transaction.Timestamp
} else {
log.Println("Replication error:", err)
}
}

PutTransaction(transaction)
}()
}
}
func (r *Replication) SendMessage(data map[string][]byte) {
if len(r.Replicas) > 0 {
r.Transmitter <- NewTransaction(data)
}
}
replication := &Replication{
Transmitter: make(chan *Transaction, 2048),
Receiver: make(chan *Transaction, 2048),
Received: make(map[string]int64),
Replicas: hosts,
}

Replication usage

go replication.Receive()
go replication.Transmit()
http.HandleFunc("/replication/receiver", func(writer http.ResponseWriter, request *http.Request) {
transaction := GetTransaction()
body, _ := ioutil.ReadAll(request.Body)
err := proto.Unmarshal(body, transaction)
if err == nil {
replication.Receiver <- transaction
fmt.Fprintln(writer, "Received transaction")
} else {
fmt.Fprintln(writer, err)
}
})
replication.SendMessage()

Conclusion

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store