From 9388b0b82129729f35bca0935db5beb03808ad9d Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Mon, 4 Sep 2023 23:00:10 +0600 Subject: [PATCH 01/33] start basic implementation --- config/config.go | 8 +++ pkg/handlers/notification.go | 129 +++++++++++++++++++++++++++++++++++ redis.yaml | 33 +++++++++ 3 files changed, 170 insertions(+) create mode 100644 redis.yaml diff --git a/config/config.go b/config/config.go index cc6702d5..eb9ed409 100644 --- a/config/config.go +++ b/config/config.go @@ -123,6 +123,9 @@ func NewDefaultConfig() *AgentConfig { Webhook: WebhookConfig{ Port: "8085", }, + Syncer: &SyncConfig{ + RedisAddr: "http://redis:6379", + }, } return &config @@ -143,6 +146,11 @@ type AgentConfig struct { Runtime RuntimeConfig `json:"runtime"` Server ServerConfig `json:"server"` Webhook WebhookConfig `json:"webhook"` + Syncer *SyncConfig `json:"syncer"` +} + +type SyncConfig struct { + RedisAddr string `json:"redisAddr"` } // HTTPSDisabledWarning is logged when keyfile and certfile are not provided in server configuration diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 55b9b7f0..c1a5e101 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -18,16 +18,44 @@ package handlers import ( + "context" "encoding/json" "fmt" "net/http" "strings" + "github.com/go-redis/redis/v8" "github.com/optimizely/agent/pkg/middleware" "github.com/optimizely/go-sdk/pkg/notification" "github.com/optimizely/go-sdk/pkg/registry" ) +type Syncer struct { +} + +func (s *Syncer) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { + return 0, nil +} + +func (s *Syncer) RemoveHandler(_ int, _ notification.Type) error { + return nil +} + +func (s *Syncer) Send(notificationType notification.Type, notification interface{}) error { + client := redis.NewClient(&redis.Options{ + Addr: "redis.demo.svc:6379", // Redis server address + Password: "", // No password + DB: 0, // Default DB + }) + defer client.Close() + + // Subscribe to a Redis channel + pubsub := client.Subscribe(context.TODO(), "notifications") + defer pubsub.Close() + + return client.Publish(context.TODO(), "notifications", notification).Err() +} + // A MessageChan is a channel of bytes // Each http handler call creates a new channel and pumps decision service messages onto it. type MessageChan chan []byte @@ -160,3 +188,104 @@ func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { } } + +// NotificationEventSteamHandler implements the http.Handler interface. +func NotificationEventSteamHandler2(w http.ResponseWriter, r *http.Request) { + // Make sure that the writer supports flushing. + flusher, ok := w.(http.Flusher) + + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } + + _, err := middleware.GetOptlyClient(r) + + if err != nil { + RenderError(err, http.StatusUnprocessableEntity, w, r) + return + } + + // Set the headers related to event streaming. + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + + // Each connection registers its own message channel with the NotificationHandler's connections registry + messageChan := make(MessageChan) + // Each connection also adds listeners + sdkKey := r.Header.Get(middleware.OptlySDKHeader) + nc := registry.GetNotificationCenter(sdkKey) + + // Parse the form. + _ = r.ParseForm() + + filters := r.Form["filter"] + + // Parse out the any filters that were added + notificationsToAdd := getFilter(filters) + + ids := []struct { + int + notification.Type + }{} + + for _, value := range notificationsToAdd { + id, e := nc.AddHandler(value, func(n interface{}) { + jsonEvent, err := json.Marshal(n) + if err != nil { + middleware.GetLogger(r).Error().Msg("encoding notification to json") + } else { + messageChan <- jsonEvent + } + }) + if e != nil { + RenderError(e, http.StatusUnprocessableEntity, w, r) + return + } + + // do defer outside the loop. + ids = append(ids, struct { + int + notification.Type + }{id, value}) + } + + // Remove the decision listener if we exited. + defer func() { + for _, id := range ids { + err := nc.RemoveHandler(id.int, id.Type) + if err != nil { + middleware.GetLogger(r).Error().AnErr("removing notification", err) + } + } + }() + + // "raw" query string option + // If provided, send raw JSON lines instead of SSE-compliant strings. + raw := len(r.Form["raw"]) > 0 + + // Listen to connection close and un-register messageChan + notify := r.Context().Done() + // block waiting or messages broadcast on this connection's messageChan + for { + select { + // Write to the ResponseWriter + case msg := <-messageChan: + if raw { + // Raw JSON events, one per line + _, _ = fmt.Fprintf(w, "%s\n", msg) + } else { + // Server Sent Events compatible + _, _ = fmt.Fprintf(w, "data: %s\n\n", msg) + } + // Flush the data immediately instead of buffering it for later. + // The flush will fail if the connection is closed. That will cause the handler to exit. + flusher.Flush() + case <-notify: + middleware.GetLogger(r).Debug().Msg("received close on the request. So, we are shutting down this handler") + return + } + } + +} diff --git a/redis.yaml b/redis.yaml new file mode 100644 index 00000000..b54c757a --- /dev/null +++ b/redis.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: demo +spec: + selector: + app: redis + ports: + - protocol: TCP + port: 6379 + targetPort: 6379 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: demo +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:latest + ports: + - containerPort: 6379 From 31de5ed8478bc7fb053c4ece555374de3f65f900 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Thu, 7 Sep 2023 23:23:46 +0600 Subject: [PATCH 02/33] resolve conflict --- pkg/handlers/notification.go | 28 ---------------------------- pkg/syncer/syncer.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 pkg/syncer/syncer.go diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index c1a5e101..05f29411 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -18,44 +18,16 @@ package handlers import ( - "context" "encoding/json" "fmt" "net/http" "strings" - "github.com/go-redis/redis/v8" "github.com/optimizely/agent/pkg/middleware" "github.com/optimizely/go-sdk/pkg/notification" "github.com/optimizely/go-sdk/pkg/registry" ) -type Syncer struct { -} - -func (s *Syncer) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { - return 0, nil -} - -func (s *Syncer) RemoveHandler(_ int, _ notification.Type) error { - return nil -} - -func (s *Syncer) Send(notificationType notification.Type, notification interface{}) error { - client := redis.NewClient(&redis.Options{ - Addr: "redis.demo.svc:6379", // Redis server address - Password: "", // No password - DB: 0, // Default DB - }) - defer client.Close() - - // Subscribe to a Redis channel - pubsub := client.Subscribe(context.TODO(), "notifications") - defer pubsub.Close() - - return client.Publish(context.TODO(), "notifications", notification).Err() -} - // A MessageChan is a channel of bytes // Each http handler call creates a new channel and pumps decision service messages onto it. type MessageChan chan []byte diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go new file mode 100644 index 00000000..64d94c22 --- /dev/null +++ b/pkg/syncer/syncer.go @@ -0,0 +1,34 @@ +package syncer + +import ( + "context" + + "github.com/go-redis/redis/v8" + "github.com/optimizely/go-sdk/pkg/notification" +) + +type CustomSyncer struct { +} + +func (s *CustomSyncer) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { + return 0, nil +} + +func (s *CustomSyncer) RemoveHandler(_ int, _ notification.Type) error { + return nil +} + +func (s *CustomSyncer) Send(notificationType notification.Type, notification interface{}) error { + client := redis.NewClient(&redis.Options{ + Addr: "redis.demo.svc:6379", // Redis server address + Password: "", // No password + DB: 0, // Default DB + }) + defer client.Close() + + // Subscribe to a Redis channel + pubsub := client.Subscribe(context.TODO(), "notifications") + defer pubsub.Close() + + return client.Publish(context.TODO(), "notifications", notification).Err() +} From 16649cd3ce3c772d729f669a8573e62ea4fe1af1 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Tue, 5 Sep 2023 20:22:45 +0600 Subject: [PATCH 03/33] change notification.go logic --- go.mod | 2 +- go.sum | 10 --- pkg/handlers/notification.go | 150 +++++++++-------------------------- 3 files changed, 40 insertions(+), 122 deletions(-) diff --git a/go.mod b/go.mod index 55826eb0..1f50c6df 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/optimizely/agent -go 1.21.0 +go 1.20 require ( github.com/go-chi/chi/v5 v5.0.8 diff --git a/go.sum b/go.sum index 3c81f704..4e590d11 100644 --- a/go.sum +++ b/go.sum @@ -75,7 +75,6 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= @@ -148,7 +147,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -198,11 +196,9 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lestrrat-go/jwx v0.9.0 h1:Fnd0EWzTm0kFrBPzE/PEPp9nzllES5buMkksPMjEKpM= github.com/lestrrat-go/jwx v0.9.0/go.mod h1:iEoxlYfZjvoGpuWwxUz+eR5e6KTJGsaRcy/YNA/UnBk= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -228,11 +224,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= -github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/optimizely/go-sdk v1.8.4-0.20230515121609-7ffed835c991 h1:bRoRDKRa7EgSTCEb54qaDuU6IDegQKQumun8buDV/cY= github.com/optimizely/go-sdk v1.8.4-0.20230515121609-7ffed835c991/go.mod h1:06VK8mwwQTEh7QzP+qivf16tXtXEpoeblqtlhfvWEgk= github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY= @@ -271,7 +264,6 @@ github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U= @@ -397,7 +389,6 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -635,7 +626,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 05f29411..c5cfe82b 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -18,11 +18,14 @@ package handlers import ( + "context" "encoding/json" "fmt" + "log" "net/http" "strings" + "github.com/go-redis/redis/v8" "github.com/optimizely/agent/pkg/middleware" "github.com/optimizely/go-sdk/pkg/notification" "github.com/optimizely/go-sdk/pkg/registry" @@ -82,8 +85,6 @@ func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") - // Each connection registers its own message channel with the NotificationHandler's connections registry - messageChan := make(MessageChan) // Each connection also adds listeners sdkKey := r.Header.Get(middleware.OptlySDKHeader) nc := registry.GetNotificationCenter(sdkKey) @@ -106,8 +107,22 @@ func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { jsonEvent, err := json.Marshal(n) if err != nil { middleware.GetLogger(r).Error().Msg("encoding notification to json") - } else { - messageChan <- jsonEvent + return + } + client := redis.NewClient(&redis.Options{ + Addr: "redis.demo.svc:6379", // Redis server address + Password: "", // No password + DB: 0, // Default DB + }) + defer client.Close() + + // Subscribe to a Redis channel + pubsub := client.Subscribe(r.Context(), "notifications") + defer pubsub.Close() + + if err := client.Publish(r.Context(), "notifications", jsonEvent).Err(); err != nil { + middleware.GetLogger(r).Err(err).Msg("failed to publish json event to pub/sub") + return } }) if e != nil { @@ -132,132 +147,45 @@ func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { } }() + client := redis.NewClient(&redis.Options{ + Addr: "redis.demo.svc:6379", // Redis server address + Password: "", // No password + DB: 0, // Default DB + }) + defer client.Close() + + // Subscribe to a Redis channel + pubsub := client.Subscribe(context.TODO(), "notifications") + defer pubsub.Close() + + // Listen to connection close and un-register messageChan + notify := r.Context().Done() + // "raw" query string option // If provided, send raw JSON lines instead of SSE-compliant strings. raw := len(r.Form["raw"]) > 0 - // Listen to connection close and un-register messageChan - notify := r.Context().Done() - // block waiting or messages broadcast on this connection's messageChan for { select { - // Write to the ResponseWriter - case msg := <-messageChan: - if raw { - // Raw JSON events, one per line - _, _ = fmt.Fprintf(w, "%s\n", msg) - } else { - // Server Sent Events compatible - _, _ = fmt.Fprintf(w, "data: %s\n\n", msg) - } - // Flush the data immediately instead of buffering it for later. - // The flush will fail if the connection is closed. That will cause the handler to exit. - flusher.Flush() case <-notify: middleware.GetLogger(r).Debug().Msg("received close on the request. So, we are shutting down this handler") return - } - } - -} - -// NotificationEventSteamHandler implements the http.Handler interface. -func NotificationEventSteamHandler2(w http.ResponseWriter, r *http.Request) { - // Make sure that the writer supports flushing. - flusher, ok := w.(http.Flusher) - - if !ok { - http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) - return - } - - _, err := middleware.GetOptlyClient(r) - - if err != nil { - RenderError(err, http.StatusUnprocessableEntity, w, r) - return - } - - // Set the headers related to event streaming. - w.Header().Set("Content-Type", "text/event-stream") - w.Header().Set("Cache-Control", "no-cache") - w.Header().Set("Connection", "keep-alive") - - // Each connection registers its own message channel with the NotificationHandler's connections registry - messageChan := make(MessageChan) - // Each connection also adds listeners - sdkKey := r.Header.Get(middleware.OptlySDKHeader) - nc := registry.GetNotificationCenter(sdkKey) - - // Parse the form. - _ = r.ParseForm() - - filters := r.Form["filter"] - - // Parse out the any filters that were added - notificationsToAdd := getFilter(filters) - - ids := []struct { - int - notification.Type - }{} - - for _, value := range notificationsToAdd { - id, e := nc.AddHandler(value, func(n interface{}) { - jsonEvent, err := json.Marshal(n) - if err != nil { - middleware.GetLogger(r).Error().Msg("encoding notification to json") - } else { - messageChan <- jsonEvent - } - }) - if e != nil { - RenderError(e, http.StatusUnprocessableEntity, w, r) - return - } - - // do defer outside the loop. - ids = append(ids, struct { - int - notification.Type - }{id, value}) - } - - // Remove the decision listener if we exited. - defer func() { - for _, id := range ids { - err := nc.RemoveHandler(id.int, id.Type) + default: + msg, err := pubsub.ReceiveMessage(r.Context()) if err != nil { - middleware.GetLogger(r).Error().AnErr("removing notification", err) + log.Println("Error receiving message:", err) + return } - } - }() - - // "raw" query string option - // If provided, send raw JSON lines instead of SSE-compliant strings. - raw := len(r.Form["raw"]) > 0 - - // Listen to connection close and un-register messageChan - notify := r.Context().Done() - // block waiting or messages broadcast on this connection's messageChan - for { - select { - // Write to the ResponseWriter - case msg := <-messageChan: if raw { // Raw JSON events, one per line - _, _ = fmt.Fprintf(w, "%s\n", msg) + _, _ = fmt.Fprintf(w, "%s\n", msg.Payload) } else { // Server Sent Events compatible - _, _ = fmt.Fprintf(w, "data: %s\n\n", msg) + _, _ = fmt.Fprintf(w, "data: %s\n\n", msg.Payload) } // Flush the data immediately instead of buffering it for later. // The flush will fail if the connection is closed. That will cause the handler to exit. flusher.Flush() - case <-notify: - middleware.GetLogger(r).Debug().Msg("received close on the request. So, we are shutting down this handler") - return } } - } From 6a36c7e43bd0eb109241f98e0c12e372df73958d Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Tue, 5 Sep 2023 21:25:02 +0600 Subject: [PATCH 04/33] modify notification handler --- config.yaml | 4 ++-- pkg/handlers/notification.go | 20 +++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/config.yaml b/config.yaml index 0e9edbbc..f0258fce 100644 --- a/config.yaml +++ b/config.yaml @@ -47,7 +47,7 @@ server: readTimeout: 5s ## the maximum duration before timing out writes of the response. ## Value can be set in seconds (e.g. "5s") or milliseconds (e.g. "5000ms") - writeTimeout: 10s + writeTimeout: -1 ## path for the health status api healthCheckPath: "/health" ## the location of the TLS key file @@ -69,7 +69,7 @@ api: ## http listener port port: "8080" ## set to true to enable subscribing to notifications via an SSE event-stream - enableNotifications: false + enableNotifications: true ## set to true to be able to override experiment bucketing. (recommended false in production) enableOverrides: true ## CORS support is provided via chi middleware diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index c5cfe82b..2551a06b 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -110,9 +110,9 @@ func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { return } client := redis.NewClient(&redis.Options{ - Addr: "redis.demo.svc:6379", // Redis server address - Password: "", // No password - DB: 0, // Default DB + Addr: "localhost:6379", // Redis server address + Password: "", // No password + DB: 0, // Default DB }) defer client.Close() @@ -148,9 +148,9 @@ func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { }() client := redis.NewClient(&redis.Options{ - Addr: "redis.demo.svc:6379", // Redis server address - Password: "", // No password - DB: 0, // Default DB + Addr: "localhost:6379", // Redis server address + Password: "", // No password + DB: 0, // Default DB }) defer client.Close() @@ -158,19 +158,17 @@ func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { pubsub := client.Subscribe(context.TODO(), "notifications") defer pubsub.Close() - // Listen to connection close and un-register messageChan - notify := r.Context().Done() - // "raw" query string option // If provided, send raw JSON lines instead of SSE-compliant strings. raw := len(r.Form["raw"]) > 0 for { select { - case <-notify: - middleware.GetLogger(r).Debug().Msg("received close on the request. So, we are shutting down this handler") + case <-r.Context().Done(): + log.Println("context cancelled, shutting down notification handler") return default: + log.Println("looking for redis message") msg, err := pubsub.ReceiveMessage(r.Context()) if err != nil { log.Println("Error receiving message:", err) From 57a4c60f3a10aa39fae8c5ca4728ba7738398f33 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Thu, 7 Sep 2023 22:32:10 +0600 Subject: [PATCH 05/33] Update config --- config.yaml | 12 ++++ config/config.go | 29 +++++---- pkg/handlers/notification.go | 111 +++++++++++++++++++++++++++++++++-- pkg/routers/api.go | 2 +- 4 files changed, 138 insertions(+), 16 deletions(-) diff --git a/config.yaml b/config.yaml index f0258fce..c5c250fd 100644 --- a/config.yaml +++ b/config.yaml @@ -205,3 +205,15 @@ runtime: ## To just read the current rate, pass rate < 0. ## (For n>1 the details of sampling may change.) mutexProfileFraction: 0 + +## synchronization should be enabled when multiple replicas of agent is deployed +## if notification synchronization is enabled, then the active notification event-stream API +## will get the notifications from multiple replicas +synchronization: + notification: + enable: true + pubsub: + redis: + host: "localhost:6379" + password: "" + database: 0 diff --git a/config/config.go b/config/config.go index eb9ed409..59f8f507 100644 --- a/config/config.go +++ b/config/config.go @@ -123,8 +123,10 @@ func NewDefaultConfig() *AgentConfig { Webhook: WebhookConfig{ Port: "8085", }, - Syncer: &SyncConfig{ - RedisAddr: "http://redis:6379", + Synchronization: SyncConfig{ + Notification: NotificationConfig{ + Enable: false, + }, }, } @@ -139,18 +141,23 @@ type AgentConfig struct { SDKKeys []string `yaml:"sdkKeys" json:"sdkKeys"` - Admin AdminConfig `json:"admin"` - API APIConfig `json:"api"` - Log LogConfig `json:"log"` - Client ClientConfig `json:"client"` - Runtime RuntimeConfig `json:"runtime"` - Server ServerConfig `json:"server"` - Webhook WebhookConfig `json:"webhook"` - Syncer *SyncConfig `json:"syncer"` + Admin AdminConfig `json:"admin"` + API APIConfig `json:"api"` + Log LogConfig `json:"log"` + Client ClientConfig `json:"client"` + Runtime RuntimeConfig `json:"runtime"` + Server ServerConfig `json:"server"` + Webhook WebhookConfig `json:"webhook"` + Synchronization SyncConfig `json:"synchronization"` } type SyncConfig struct { - RedisAddr string `json:"redisAddr"` + Notification NotificationConfig `json:"notification"` +} + +type NotificationConfig struct { + Enable bool `json:"enable"` + Pubsub map[string]interface{} `json:"pubsub"` } // HTTPSDisabledWarning is logged when keyfile and certfile are not provided in server configuration diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 2551a06b..1c55b224 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -18,7 +18,6 @@ package handlers import ( - "context" "encoding/json" "fmt" "log" @@ -85,6 +84,107 @@ func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") + // Each connection registers its own message channel with the NotificationHandler's connections registry + messageChan := make(MessageChan) + // Each connection also adds listeners + sdkKey := r.Header.Get(middleware.OptlySDKHeader) + nc := registry.GetNotificationCenter(sdkKey) + + // Parse the form. + _ = r.ParseForm() + + filters := r.Form["filter"] + + // Parse out the any filters that were added + notificationsToAdd := getFilter(filters) + + ids := []struct { + int + notification.Type + }{} + + for _, value := range notificationsToAdd { + id, e := nc.AddHandler(value, func(n interface{}) { + jsonEvent, err := json.Marshal(n) + if err != nil { + middleware.GetLogger(r).Error().Msg("encoding notification to json") + } else { + messageChan <- jsonEvent + } + }) + if e != nil { + RenderError(e, http.StatusUnprocessableEntity, w, r) + return + } + + // do defer outside the loop. + ids = append(ids, struct { + int + notification.Type + }{id, value}) + } + + // Remove the decision listener if we exited. + defer func() { + for _, id := range ids { + err := nc.RemoveHandler(id.int, id.Type) + if err != nil { + middleware.GetLogger(r).Error().AnErr("removing notification", err) + } + } + }() + + // "raw" query string option + // If provided, send raw JSON lines instead of SSE-compliant strings. + raw := len(r.Form["raw"]) > 0 + + // Listen to connection close and un-register messageChan + notify := r.Context().Done() + // block waiting or messages broadcast on this connection's messageChan + for { + select { + // Write to the ResponseWriter + case msg := <-messageChan: + if raw { + // Raw JSON events, one per line + _, _ = fmt.Fprintf(w, "%s\n", msg) + } else { + // Server Sent Events compatible + _, _ = fmt.Fprintf(w, "data: %s\n\n", msg) + } + // Flush the data immediately instead of buffering it for later. + // The flush will fail if the connection is closed. That will cause the handler to exit. + flusher.Flush() + case <-notify: + middleware.GetLogger(r).Debug().Msg("received close on the request. So, we are shutting down this handler") + return + } + } + +} + +// NotificationEventSteamHandler implements the http.Handler interface. +func NotificationEventSteamSyncHandler(w http.ResponseWriter, r *http.Request) { + // Make sure that the writer supports flushing. + flusher, ok := w.(http.Flusher) + + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } + + _, err := middleware.GetOptlyClient(r) + + if err != nil { + RenderError(err, http.StatusUnprocessableEntity, w, r) + return + } + + // Set the headers related to event streaming. + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + // Each connection also adds listeners sdkKey := r.Header.Get(middleware.OptlySDKHeader) nc := registry.GetNotificationCenter(sdkKey) @@ -155,17 +255,20 @@ func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { defer client.Close() // Subscribe to a Redis channel - pubsub := client.Subscribe(context.TODO(), "notifications") + pubsub := client.Subscribe(r.Context(), "notifications") defer pubsub.Close() // "raw" query string option // If provided, send raw JSON lines instead of SSE-compliant strings. raw := len(r.Form["raw"]) > 0 + // Listen to connection close and un-register messageChan + notify := r.Context().Done() + for { select { - case <-r.Context().Done(): - log.Println("context cancelled, shutting down notification handler") + case <-notify: + middleware.GetLogger(r).Debug().Msg("received close on the request. So, we are shutting down this handler") return default: log.Println("looking for redis message") diff --git a/pkg/routers/api.go b/pkg/routers/api.go index c2e513c8..7f73b5bc 100644 --- a/pkg/routers/api.go +++ b/pkg/routers/api.go @@ -82,7 +82,7 @@ func NewDefaultAPIRouter(optlyCache optimizely.Cache, conf config.APIConfig, met overrideHandler = forbiddenHandler("Overrides not enabled") } - nStreamHandler := handlers.NotificationEventSteamHandler + nStreamHandler := handlers.NotificationEventSteamSyncHandler if !conf.EnableNotifications { nStreamHandler = forbiddenHandler("Notification stream not enabled") } From c3b8bc8e8a3d779e74ce89f2a3e5e9410e8dda33 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Fri, 8 Sep 2023 17:42:59 +0600 Subject: [PATCH 06/33] add redis pubsub struct --- cmd/optimizely/main.go | 2 +- config.yaml | 4 +- config/config.go | 10 +- pkg/handlers/notification.go | 237 ++++++++++++++++++++--------------- pkg/routers/api.go | 16 +-- 5 files changed, 157 insertions(+), 112 deletions(-) diff --git a/cmd/optimizely/main.go b/cmd/optimizely/main.go index 1b9c4002..9fd18fe3 100644 --- a/cmd/optimizely/main.go +++ b/cmd/optimizely/main.go @@ -164,7 +164,7 @@ func main() { cancel() }() - apiRouter := routers.NewDefaultAPIRouter(optlyCache, conf.API, agentMetricsRegistry) + apiRouter := routers.NewDefaultAPIRouter(optlyCache, conf, agentMetricsRegistry) adminRouter := routers.NewAdminRouter(*conf) log.Info().Str("version", conf.Version).Msg("Starting services.") diff --git a/config.yaml b/config.yaml index c5c250fd..04d1fb78 100644 --- a/config.yaml +++ b/config.yaml @@ -214,6 +214,6 @@ synchronization: enable: true pubsub: redis: - host: "localhost:6379" + addr: "localhost:6379" password: "" - database: 0 + db: 0 diff --git a/config/config.go b/config/config.go index 59f8f507..1f4100e2 100644 --- a/config/config.go +++ b/config/config.go @@ -156,8 +156,14 @@ type SyncConfig struct { } type NotificationConfig struct { - Enable bool `json:"enable"` - Pubsub map[string]interface{} `json:"pubsub"` + Enable bool `json:"enable"` + Pubsub RedisConfig `json:"pubsub"` +} + +type RedisConfig struct { + Addr string `json:"addr"` + Password string `json:"password"` + DB int `json:"db"` } // HTTPSDisabledWarning is logged when keyfile and certfile are not provided in server configuration diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 1c55b224..d37b9eb1 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -18,6 +18,7 @@ package handlers import ( + "context" "encoding/json" "fmt" "log" @@ -25,9 +26,11 @@ import ( "strings" "github.com/go-redis/redis/v8" + "github.com/optimizely/agent/config" "github.com/optimizely/agent/pkg/middleware" "github.com/optimizely/go-sdk/pkg/notification" "github.com/optimizely/go-sdk/pkg/registry" + "github.com/rs/zerolog" ) // A MessageChan is a channel of bytes @@ -62,6 +65,13 @@ func getFilter(filters []string) map[string]notification.Type { return notificationsToAdd } +func NotificationEventStreamDecider(conf *config.SyncConfig) http.HandlerFunc { + if !conf.Notification.Enable { + return NotificationEventSteamHandler + } + return NotificationEventSteamSyncHandler(conf) +} + // NotificationEventSteamHandler implements the http.Handler interface. func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { // Make sure that the writer supports flushing. @@ -163,130 +173,159 @@ func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { } -// NotificationEventSteamHandler implements the http.Handler interface. -func NotificationEventSteamSyncHandler(w http.ResponseWriter, r *http.Request) { - // Make sure that the writer supports flushing. - flusher, ok := w.(http.Flusher) - - if !ok { - http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) - return - } +const ( + PubSubChan = "optimizely-notifications" + PubSubRedis = "redis" +) - _, err := middleware.GetOptlyClient(r) +type RedisPubSubSyncer struct { + Addr string + Password string + DB int + logger *zerolog.Logger +} - if err != nil { - RenderError(err, http.StatusUnprocessableEntity, w, r) - return +func NewRedisPubSubSyncer(logger *zerolog.Logger, conf *config.SyncConfig) *RedisPubSubSyncer { + return &RedisPubSubSyncer{ + Addr: conf.Notification.Pubsub.Addr, + Password: conf.Notification.Pubsub.Password, + DB: conf.Notification.Pubsub.DB, + logger: logger, } +} - // Set the headers related to event streaming. - w.Header().Set("Content-Type", "text/event-stream") - w.Header().Set("Cache-Control", "no-cache") - w.Header().Set("Connection", "keep-alive") +func (r *RedisPubSubSyncer) GetNotificationSyncer(ctx context.Context) func(n interface{}) { + return func(n interface{}) { + jsonEvent, err := json.Marshal(n) + if err != nil { + r.logger.Error().Msg("encoding notification to json") + return + } + client := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", // Redis server address + Password: "", // No password + DB: 0, // Default DB + }) + defer client.Close() - // Each connection also adds listeners - sdkKey := r.Header.Get(middleware.OptlySDKHeader) - nc := registry.GetNotificationCenter(sdkKey) + // Subscribe to a Redis channel + pubsub := client.Subscribe(ctx, PubSubChan) + defer pubsub.Close() - // Parse the form. - _ = r.ParseForm() + if err := client.Publish(ctx, PubSubChan, jsonEvent).Err(); err != nil { + r.logger.Err(err).Msg("failed to publish json event to pub/sub") + return + } + } +} - filters := r.Form["filter"] +// NotificationEventSteamHandler implements the http.Handler interface. +func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Make sure that the writer supports flushing. + flusher, ok := w.(http.Flusher) - // Parse out the any filters that were added - notificationsToAdd := getFilter(filters) + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } - ids := []struct { - int - notification.Type - }{} + _, err := middleware.GetOptlyClient(r) - for _, value := range notificationsToAdd { - id, e := nc.AddHandler(value, func(n interface{}) { - jsonEvent, err := json.Marshal(n) - if err != nil { - middleware.GetLogger(r).Error().Msg("encoding notification to json") - return - } - client := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", // Redis server address - Password: "", // No password - DB: 0, // Default DB - }) - defer client.Close() - - // Subscribe to a Redis channel - pubsub := client.Subscribe(r.Context(), "notifications") - defer pubsub.Close() - - if err := client.Publish(r.Context(), "notifications", jsonEvent).Err(); err != nil { - middleware.GetLogger(r).Err(err).Msg("failed to publish json event to pub/sub") - return - } - }) - if e != nil { - RenderError(e, http.StatusUnprocessableEntity, w, r) + if err != nil { + RenderError(err, http.StatusUnprocessableEntity, w, r) return } - // do defer outside the loop. - ids = append(ids, struct { + // Set the headers related to event streaming. + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + + // Each connection also adds listeners + sdkKey := r.Header.Get(middleware.OptlySDKHeader) + nc := registry.GetNotificationCenter(sdkKey) + + // Parse the form. + _ = r.ParseForm() + + filters := r.Form["filter"] + + // Parse out the any filters that were added + notificationsToAdd := getFilter(filters) + + ids := []struct { int notification.Type - }{id, value}) - } + }{} - // Remove the decision listener if we exited. - defer func() { - for _, id := range ids { - err := nc.RemoveHandler(id.int, id.Type) - if err != nil { - middleware.GetLogger(r).Error().AnErr("removing notification", err) + syncer := NewRedisPubSubSyncer(middleware.GetLogger(r), conf) + + for _, value := range notificationsToAdd { + id, e := nc.AddHandler(value, syncer.GetNotificationSyncer(r.Context())) + if e != nil { + RenderError(e, http.StatusUnprocessableEntity, w, r) + return } + + // do defer outside the loop. + ids = append(ids, struct { + int + notification.Type + }{id, value}) } - }() - client := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", // Redis server address - Password: "", // No password - DB: 0, // Default DB - }) - defer client.Close() + // Remove the decision listener if we exited. + defer func() { + for _, id := range ids { + err := nc.RemoveHandler(id.int, id.Type) + if err != nil { + middleware.GetLogger(r).Error().AnErr("removing notification", err) + } + } + }() - // Subscribe to a Redis channel - pubsub := client.Subscribe(r.Context(), "notifications") - defer pubsub.Close() + client := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", // Redis server address + Password: "", // No password + DB: 0, // Default DB + }) + defer client.Close() - // "raw" query string option - // If provided, send raw JSON lines instead of SSE-compliant strings. - raw := len(r.Form["raw"]) > 0 + // Subscribe to a Redis channel + pubsub := client.Subscribe(r.Context(), "notifications") + defer pubsub.Close() - // Listen to connection close and un-register messageChan - notify := r.Context().Done() + // "raw" query string option + // If provided, send raw JSON lines instead of SSE-compliant strings. + raw := len(r.Form["raw"]) > 0 - for { - select { - case <-notify: - middleware.GetLogger(r).Debug().Msg("received close on the request. So, we are shutting down this handler") - return - default: - log.Println("looking for redis message") - msg, err := pubsub.ReceiveMessage(r.Context()) - if err != nil { - log.Println("Error receiving message:", err) + // Listen to connection close and un-register messageChan + notify := r.Context().Done() + + for { + select { + case <-notify: + middleware.GetLogger(r).Debug().Msg("received close on the request. So, we are shutting down this handler") return + default: + log.Println("looking for redis message") + msg, err := pubsub.ReceiveMessage(r.Context()) + if err != nil { + log.Println("Error receiving message:", err) + return + } + if raw { + // Raw JSON events, one per line + _, _ = fmt.Fprintf(w, "%s\n", msg.Payload) + } else { + // Server Sent Events compatible + _, _ = fmt.Fprintf(w, "data: %s\n\n", msg.Payload) + } + // Flush the data immediately instead of buffering it for later. + // The flush will fail if the connection is closed. That will cause the handler to exit. + flusher.Flush() } - if raw { - // Raw JSON events, one per line - _, _ = fmt.Fprintf(w, "%s\n", msg.Payload) - } else { - // Server Sent Events compatible - _, _ = fmt.Fprintf(w, "data: %s\n\n", msg.Payload) - } - // Flush the data immediately instead of buffering it for later. - // The flush will fail if the connection is closed. That will cause the handler to exit. - flusher.Flush() } } } diff --git a/pkg/routers/api.go b/pkg/routers/api.go index 7f73b5bc..4a68ba9f 100644 --- a/pkg/routers/api.go +++ b/pkg/routers/api.go @@ -63,35 +63,35 @@ func forbiddenHandler(message string) http.HandlerFunc { } // NewDefaultAPIRouter creates a new router with the default backing optimizely.Cache -func NewDefaultAPIRouter(optlyCache optimizely.Cache, conf config.APIConfig, metricsRegistry *metrics.Registry) http.Handler { +func NewDefaultAPIRouter(optlyCache optimizely.Cache, conf *config.AgentConfig, metricsRegistry *metrics.Registry) http.Handler { - authProvider := middleware.NewAuth(&conf.Auth) + authProvider := middleware.NewAuth(&conf.API.Auth) if authProvider == nil { log.Error().Msg("unable to initialize api auth middleware.") return nil } - authHandler := handlers.NewOAuthHandler(&conf.Auth) + authHandler := handlers.NewOAuthHandler(&conf.API.Auth) if authHandler == nil { log.Error().Msg("unable to initialize api auth handler.") return nil } overrideHandler := handlers.Override - if !conf.EnableOverrides { + if !conf.API.EnableOverrides { overrideHandler = forbiddenHandler("Overrides not enabled") } - nStreamHandler := handlers.NotificationEventSteamSyncHandler - if !conf.EnableNotifications { + nStreamHandler := handlers.NotificationEventStreamDecider(&conf.Synchronization) + if !conf.API.EnableNotifications { nStreamHandler = forbiddenHandler("Notification stream not enabled") } mw := middleware.CachedOptlyMiddleware{Cache: optlyCache} - corsHandler := createCorsHandler(conf.CORS) + corsHandler := createCorsHandler(conf.API.CORS) spec := &APIOptions{ - maxConns: conf.MaxConns, + maxConns: conf.API.MaxConns, metricsRegistry: metricsRegistry, configHandler: handlers.OptimizelyConfig, datafileHandler: handlers.GetDatafile, From 97dffaef4daa38ad8804988d16329fad3fa0c232 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Fri, 8 Sep 2023 17:59:22 +0600 Subject: [PATCH 07/33] update config --- config.yaml | 7 +++---- pkg/handlers/notification.go | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/config.yaml b/config.yaml index 04d1fb78..a182394d 100644 --- a/config.yaml +++ b/config.yaml @@ -213,7 +213,6 @@ synchronization: notification: enable: true pubsub: - redis: - addr: "localhost:6379" - password: "" - db: 0 + addr: "localhost:6379" + password: "" + db: 0 diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index d37b9eb1..3597ac71 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -286,9 +286,9 @@ func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc }() client := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", // Redis server address - Password: "", // No password - DB: 0, // Default DB + Addr: conf.Notification.Pubsub.Addr, // Redis server address + Password: conf.Notification.Pubsub.Password, // No password + DB: conf.Notification.Pubsub.DB, // Default DB }) defer client.Close() From bfe5b7c98bc598bf59db1b8660fc68396959885a Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Mon, 11 Sep 2023 18:25:11 +0600 Subject: [PATCH 08/33] implement & use redis center as notification.center --- Dockerfile | 14 ++++++ cmd/optimizely/main.go | 4 +- config.yaml | 2 +- go.mod | 4 +- go.sum | 14 +++++- pkg/handlers/notification.go | 94 +----------------------------------- pkg/optimizely/cache.go | 20 ++++++-- pkg/syncer/syncer.go | 92 +++++++++++++++++++++++++++++++---- 8 files changed, 132 insertions(+), 112 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..5b39880b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM golang:1.21.0 as builder +RUN addgroup -u 1000 agentgroup &&\ + useradd -u 1000 agentuser -g agentgroup +WORKDIR /go/src/github.com/optimizely/agent +COPY . . +RUN make setup build &&\ + make ci_build_static_binary + +FROM scratch +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /go/src/github.com/optimizely/agent/bin/optimizely /optimizely +COPY --from=builder /etc/passwd /etc/passwd +USER agentuser +CMD ["/optimizely"] \ No newline at end of file diff --git a/cmd/optimizely/main.go b/cmd/optimizely/main.go index 9fd18fe3..f744bfb0 100644 --- a/cmd/optimizely/main.go +++ b/cmd/optimizely/main.go @@ -150,8 +150,8 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) // Create default service context sg := server.NewGroup(ctx, conf.Server) // Create a new server group to manage the individual http listeners - optlyCache := optimizely.NewCache(ctx, conf.Client, sdkMetricsRegistry) - optlyCache.Init(conf.SDKKeys) + optlyCache := optimizely.NewCache(ctx, conf, sdkMetricsRegistry) + optlyCache.Init(conf) // goroutine to check for signals to gracefully shutdown listeners go func() { diff --git a/config.yaml b/config.yaml index a182394d..4f688e13 100644 --- a/config.yaml +++ b/config.yaml @@ -213,6 +213,6 @@ synchronization: notification: enable: true pubsub: - addr: "localhost:6379" + addr: "redis.demo.svc:6379" password: "" db: 0 diff --git a/go.mod b/go.mod index 1f50c6df..e38a9d18 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/optimizely/agent -go 1.20 +go 1.21.0 require ( github.com/go-chi/chi/v5 v5.0.8 @@ -12,7 +12,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.3.0 github.com/lestrrat-go/jwx v0.9.0 - github.com/optimizely/go-sdk v1.8.4-0.20230515121609-7ffed835c991 + github.com/optimizely/go-sdk v1.8.4-0.20230905115300-bf10312aeb60 github.com/orcaman/concurrent-map v1.0.0 github.com/prometheus/client_golang v1.11.0 github.com/rakyll/statik v0.1.7 diff --git a/go.sum b/go.sum index 4e590d11..1a3e2577 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= @@ -147,6 +148,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -196,9 +198,11 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lestrrat-go/jwx v0.9.0 h1:Fnd0EWzTm0kFrBPzE/PEPp9nzllES5buMkksPMjEKpM= github.com/lestrrat-go/jwx v0.9.0/go.mod h1:iEoxlYfZjvoGpuWwxUz+eR5e6KTJGsaRcy/YNA/UnBk= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -224,10 +228,13 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= -github.com/optimizely/go-sdk v1.8.4-0.20230515121609-7ffed835c991 h1:bRoRDKRa7EgSTCEb54qaDuU6IDegQKQumun8buDV/cY= -github.com/optimizely/go-sdk v1.8.4-0.20230515121609-7ffed835c991/go.mod h1:06VK8mwwQTEh7QzP+qivf16tXtXEpoeblqtlhfvWEgk= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/optimizely/go-sdk v1.8.4-0.20230905115300-bf10312aeb60 h1:67FgE2f4on8nhKzvUf13xvQ1Qny1o2iZp4/xL701f8o= +github.com/optimizely/go-sdk v1.8.4-0.20230905115300-bf10312aeb60/go.mod h1:zITWqffjOXsae/Z0PlCN5kgJRgJF/0g/k8RBEsxNrxg= github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY= github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= @@ -264,6 +271,7 @@ github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U= @@ -389,6 +397,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -626,6 +635,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 3597ac71..ee1086e9 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -18,7 +18,6 @@ package handlers import ( - "context" "encoding/json" "fmt" "log" @@ -28,9 +27,9 @@ import ( "github.com/go-redis/redis/v8" "github.com/optimizely/agent/config" "github.com/optimizely/agent/pkg/middleware" + "github.com/optimizely/agent/pkg/syncer" "github.com/optimizely/go-sdk/pkg/notification" "github.com/optimizely/go-sdk/pkg/registry" - "github.com/rs/zerolog" ) // A MessageChan is a channel of bytes @@ -173,52 +172,6 @@ func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { } -const ( - PubSubChan = "optimizely-notifications" - PubSubRedis = "redis" -) - -type RedisPubSubSyncer struct { - Addr string - Password string - DB int - logger *zerolog.Logger -} - -func NewRedisPubSubSyncer(logger *zerolog.Logger, conf *config.SyncConfig) *RedisPubSubSyncer { - return &RedisPubSubSyncer{ - Addr: conf.Notification.Pubsub.Addr, - Password: conf.Notification.Pubsub.Password, - DB: conf.Notification.Pubsub.DB, - logger: logger, - } -} - -func (r *RedisPubSubSyncer) GetNotificationSyncer(ctx context.Context) func(n interface{}) { - return func(n interface{}) { - jsonEvent, err := json.Marshal(n) - if err != nil { - r.logger.Error().Msg("encoding notification to json") - return - } - client := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", // Redis server address - Password: "", // No password - DB: 0, // Default DB - }) - defer client.Close() - - // Subscribe to a Redis channel - pubsub := client.Subscribe(ctx, PubSubChan) - defer pubsub.Close() - - if err := client.Publish(ctx, PubSubChan, jsonEvent).Err(); err != nil { - r.logger.Err(err).Msg("failed to publish json event to pub/sub") - return - } - } -} - // NotificationEventSteamHandler implements the http.Handler interface. func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -242,49 +195,6 @@ func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") - // Each connection also adds listeners - sdkKey := r.Header.Get(middleware.OptlySDKHeader) - nc := registry.GetNotificationCenter(sdkKey) - - // Parse the form. - _ = r.ParseForm() - - filters := r.Form["filter"] - - // Parse out the any filters that were added - notificationsToAdd := getFilter(filters) - - ids := []struct { - int - notification.Type - }{} - - syncer := NewRedisPubSubSyncer(middleware.GetLogger(r), conf) - - for _, value := range notificationsToAdd { - id, e := nc.AddHandler(value, syncer.GetNotificationSyncer(r.Context())) - if e != nil { - RenderError(e, http.StatusUnprocessableEntity, w, r) - return - } - - // do defer outside the loop. - ids = append(ids, struct { - int - notification.Type - }{id, value}) - } - - // Remove the decision listener if we exited. - defer func() { - for _, id := range ids { - err := nc.RemoveHandler(id.int, id.Type) - if err != nil { - middleware.GetLogger(r).Error().AnErr("removing notification", err) - } - } - }() - client := redis.NewClient(&redis.Options{ Addr: conf.Notification.Pubsub.Addr, // Redis server address Password: conf.Notification.Pubsub.Password, // No password @@ -293,7 +203,7 @@ func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc defer client.Close() // Subscribe to a Redis channel - pubsub := client.Subscribe(r.Context(), "notifications") + pubsub := client.Subscribe(r.Context(), syncer.PubSubChan) defer pubsub.Close() // "raw" query string option diff --git a/pkg/optimizely/cache.go b/pkg/optimizely/cache.go index d9af8388..9df9c7fb 100644 --- a/pkg/optimizely/cache.go +++ b/pkg/optimizely/cache.go @@ -26,6 +26,7 @@ import ( "sync" "github.com/optimizely/agent/config" + "github.com/optimizely/agent/pkg/syncer" "github.com/optimizely/agent/plugins/odpcache" "github.com/optimizely/agent/plugins/userprofileservice" "github.com/optimizely/go-sdk/pkg/client" @@ -33,13 +34,16 @@ import ( "github.com/optimizely/go-sdk/pkg/decision" "github.com/optimizely/go-sdk/pkg/event" "github.com/optimizely/go-sdk/pkg/logging" + "github.com/optimizely/go-sdk/pkg/notification" "github.com/optimizely/go-sdk/pkg/odp" odpEventPkg "github.com/optimizely/go-sdk/pkg/odp/event" odpSegmentPkg "github.com/optimizely/go-sdk/pkg/odp/segment" + "github.com/optimizely/go-sdk/pkg/registry" "github.com/optimizely/go-sdk/pkg/utils" odpCachePkg "github.com/optimizely/go-sdk/pkg/odp/cache" cmap "github.com/orcaman/concurrent-map" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) @@ -61,7 +65,7 @@ type OptlyCache struct { } // NewCache returns a new implementation of OptlyCache interface backed by a concurrent map. -func NewCache(ctx context.Context, conf config.ClientConfig, metricsRegistry *MetricsRegistry) *OptlyCache { +func NewCache(ctx context.Context, conf *config.AgentConfig, metricsRegistry *MetricsRegistry) *OptlyCache { // TODO is there a cleaner way to handle this translation??? cmLoader := func(sdkkey string, options ...sdkconfig.OptionFunc) SyncedConfigManager { @@ -83,8 +87,8 @@ func NewCache(ctx context.Context, conf config.ClientConfig, metricsRegistry *Me } // Init takes a slice of sdkKeys to warm the cache upon startup -func (c *OptlyCache) Init(sdkKeys []string) { - for _, sdkKey := range sdkKeys { +func (c *OptlyCache) Init(conf *config.AgentConfig) { + for _, sdkKey := range conf.SDKKeys { if _, err := c.GetClient(sdkKey); err != nil { message := "Failed to initialize Optimizely Client." if ShouldIncludeSDKKey { @@ -93,6 +97,9 @@ func (c *OptlyCache) Init(sdkKeys []string) { } log.Warn().Msg(message) } + + nc := registry.GetNotificationCenter(sdkKey) + nc.AddHandler(notification.Track, syncer.NewRedisPubSubSyncer(&zerolog.Logger{}, &conf.Synchronization).GetNotificationSyncer(context.TODO())) } } @@ -168,12 +175,13 @@ func regexValidator(sdkKeyRegex string) func(string) bool { } func defaultLoader( - conf config.ClientConfig, + conff *config.AgentConfig, metricsRegistry *MetricsRegistry, userProfileServiceMap cmap.ConcurrentMap, odpCacheMap cmap.ConcurrentMap, pcFactory func(sdkKey string, options ...sdkconfig.OptionFunc) SyncedConfigManager, bpFactory func(options ...event.BPOptionConfig) *event.BatchEventProcessor) func(clientKey string) (*OptlyClient, error) { + conf := conff.Client validator := regexValidator(conf.SdkKeyRegex) return func(clientKey string) (*OptlyClient, error) { @@ -246,6 +254,7 @@ func defaultLoader( client.WithExperimentOverrides(forcedVariations), client.WithEventProcessor(ep), client.WithOdpDisabled(conf.ODP.Disable), + client.WithNotificationCenter(syncer.NewRedisCenter(&conff.Synchronization)), } var clientUserProfileService decision.UserProfileService @@ -300,6 +309,9 @@ func defaultLoader( optimizelyClient, err := optimizelyFactory.Client( clientOptions..., ) + if err != nil { + return nil, err + } return &OptlyClient{optimizelyClient, configManager, forcedVariations, clientUserProfileService, clientODPCache}, err } } diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index 64d94c22..3d2ebabc 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -2,33 +2,107 @@ package syncer import ( "context" + "encoding/json" "github.com/go-redis/redis/v8" + "github.com/optimizely/agent/config" "github.com/optimizely/go-sdk/pkg/notification" + "github.com/rs/zerolog" ) -type CustomSyncer struct { +const ( + PubSubChan = "optimizely-notifications" + PubSubRedis = "redis" +) + +type RedisCenter struct { + Addr string + Password string + DB int + logger *zerolog.Logger } -func (s *CustomSyncer) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { +// AddHandler(Type, func(interface{})) (int, error) +// RemoveHandler(int, Type) error +// Send(Type, interface{}) error + +func NewRedisCenter(conf *config.SyncConfig) *RedisCenter { + return &RedisCenter{ + Addr: conf.Notification.Pubsub.Addr, + Password: conf.Notification.Pubsub.Password, + DB: conf.Notification.Pubsub.DB, + } +} + +func (r *RedisCenter) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { return 0, nil } -func (s *CustomSyncer) RemoveHandler(_ int, _ notification.Type) error { +func (r *RedisCenter) RemoveHandler(_ int, t notification.Type) error { return nil } -func (s *CustomSyncer) Send(notificationType notification.Type, notification interface{}) error { +func (r *RedisCenter) Send(_ notification.Type, n interface{}) error { + jsonEvent, err := json.Marshal(n) + if err != nil { + r.logger.Error().Msg("encoding notification to json") + return err + } client := redis.NewClient(&redis.Options{ - Addr: "redis.demo.svc:6379", // Redis server address - Password: "", // No password - DB: 0, // Default DB + Addr: r.Addr, // Redis server address + Password: r.Password, // No password + DB: r.DB, // Default DB }) defer client.Close() // Subscribe to a Redis channel - pubsub := client.Subscribe(context.TODO(), "notifications") + pubsub := client.Subscribe(context.TODO(), PubSubChan) defer pubsub.Close() - return client.Publish(context.TODO(), "notifications", notification).Err() + if err := client.Publish(context.TODO(), PubSubChan, jsonEvent).Err(); err != nil { + r.logger.Err(err).Msg("failed to publish json event to pub/sub") + return err + } + return nil +} + +type RedisPubSubSyncer struct { + Addr string + Password string + DB int + logger *zerolog.Logger +} + +func NewRedisPubSubSyncer(logger *zerolog.Logger, conf *config.SyncConfig) *RedisPubSubSyncer { + return &RedisPubSubSyncer{ + Addr: conf.Notification.Pubsub.Addr, + Password: conf.Notification.Pubsub.Password, + DB: conf.Notification.Pubsub.DB, + logger: logger, + } +} + +func (r *RedisPubSubSyncer) GetNotificationSyncer(ctx context.Context) func(n interface{}) { + return func(n interface{}) { + jsonEvent, err := json.Marshal(n) + if err != nil { + r.logger.Error().Msg("encoding notification to json") + return + } + client := redis.NewClient(&redis.Options{ + Addr: r.Addr, // Redis server address + Password: r.Password, // No password + DB: r.DB, // Default DB + }) + defer client.Close() + + // Subscribe to a Redis channel + pubsub := client.Subscribe(ctx, PubSubChan) + defer pubsub.Close() + + if err := client.Publish(ctx, PubSubChan, jsonEvent).Err(); err != nil { + r.logger.Err(err).Msg("failed to publish json event to pub/sub") + return + } + } } From 6669a59066397fd6c48eed842c5965a0dc05d7da Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Mon, 11 Sep 2023 23:47:14 +0600 Subject: [PATCH 09/33] use opticlient from middleware --- go.mod | 2 +- go.sum | 4 ++-- pkg/handlers/notification.go | 42 ++++++++++++++++++++++++++++++++++-- pkg/optimizely/cache.go | 4 +++- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index e38a9d18..43fd12ad 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.3.0 github.com/lestrrat-go/jwx v0.9.0 - github.com/optimizely/go-sdk v1.8.4-0.20230905115300-bf10312aeb60 + github.com/optimizely/go-sdk v1.8.4-0.20230911163718-b10e161e39b8 github.com/orcaman/concurrent-map v1.0.0 github.com/prometheus/client_golang v1.11.0 github.com/rakyll/statik v0.1.7 diff --git a/go.sum b/go.sum index 1a3e2577..bc9a6ef9 100644 --- a/go.sum +++ b/go.sum @@ -233,8 +233,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/optimizely/go-sdk v1.8.4-0.20230905115300-bf10312aeb60 h1:67FgE2f4on8nhKzvUf13xvQ1Qny1o2iZp4/xL701f8o= -github.com/optimizely/go-sdk v1.8.4-0.20230905115300-bf10312aeb60/go.mod h1:zITWqffjOXsae/Z0PlCN5kgJRgJF/0g/k8RBEsxNrxg= +github.com/optimizely/go-sdk v1.8.4-0.20230911163718-b10e161e39b8 h1:1LhZsu7IB7LR3PzwIzfP56cdOkUAKRXxW1wljd352sg= +github.com/optimizely/go-sdk v1.8.4-0.20230911163718-b10e161e39b8/go.mod h1:zITWqffjOXsae/Z0PlCN5kgJRgJF/0g/k8RBEsxNrxg= github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY= github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index ee1086e9..b408a59f 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -183,18 +183,56 @@ func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc return } - _, err := middleware.GetOptlyClient(r) - + optlyClient, err := middleware.GetOptlyClient(r) if err != nil { RenderError(err, http.StatusUnprocessableEntity, w, r) return } + nc := optlyClient.GetNotificationCenter() + // Set the headers related to event streaming. w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") + // Parse the form. + _ = r.ParseForm() + + filters := r.Form["filter"] + + // Parse out the any filters that were added + notificationsToAdd := getFilter(filters) + + ids := []struct { + int + notification.Type + }{} + + for _, value := range notificationsToAdd { + id, e := nc.AddHandler(value, syncer.NewRedisPubSubSyncer(middleware.GetLogger(r), conf).GetNotificationSyncer(r.Context())) + if e != nil { + RenderError(e, http.StatusUnprocessableEntity, w, r) + return + } + + // do defer outside the loop. + ids = append(ids, struct { + int + notification.Type + }{id, value}) + } + + // Remove the decision listener if we exited. + defer func() { + for _, id := range ids { + err := nc.RemoveHandler(id.int, id.Type) + if err != nil { + middleware.GetLogger(r).Error().AnErr("removing notification", err) + } + } + }() + client := redis.NewClient(&redis.Options{ Addr: conf.Notification.Pubsub.Addr, // Redis server address Password: conf.Notification.Pubsub.Password, // No password diff --git a/pkg/optimizely/cache.go b/pkg/optimizely/cache.go index 9df9c7fb..641da908 100644 --- a/pkg/optimizely/cache.go +++ b/pkg/optimizely/cache.go @@ -254,7 +254,9 @@ func defaultLoader( client.WithExperimentOverrides(forcedVariations), client.WithEventProcessor(ep), client.WithOdpDisabled(conf.ODP.Disable), - client.WithNotificationCenter(syncer.NewRedisCenter(&conff.Synchronization)), + } + if conff.Synchronization.Notification.Enable { + clientOptions = append(clientOptions, client.WithNotificationCenter(syncer.NewRedisCenter(&conff.Synchronization))) } var clientUserProfileService decision.UserProfileService From eeeb53bc3b50ffe2eeeaa7b8223699573d92a0a2 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Tue, 12 Sep 2023 16:40:21 +0600 Subject: [PATCH 10/33] rename notification handler name --- pkg/handlers/notification.go | 10 +++++----- pkg/routers/api.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index b408a59f..8eef49a6 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -64,15 +64,15 @@ func getFilter(filters []string) map[string]notification.Type { return notificationsToAdd } -func NotificationEventStreamDecider(conf *config.SyncConfig) http.HandlerFunc { - if !conf.Notification.Enable { - return NotificationEventSteamHandler +func NotificationEventStreamHandler(syncConfig *config.SyncConfig) http.HandlerFunc { + if !syncConfig.Notification.Enable { + return NotificationEventSteamMonolithHandler } - return NotificationEventSteamSyncHandler(conf) + return NotificationEventSteamSyncHandler(syncConfig) } // NotificationEventSteamHandler implements the http.Handler interface. -func NotificationEventSteamHandler(w http.ResponseWriter, r *http.Request) { +func NotificationEventSteamMonolithHandler(w http.ResponseWriter, r *http.Request) { // Make sure that the writer supports flushing. flusher, ok := w.(http.Flusher) diff --git a/pkg/routers/api.go b/pkg/routers/api.go index 4a68ba9f..759072bf 100644 --- a/pkg/routers/api.go +++ b/pkg/routers/api.go @@ -82,7 +82,7 @@ func NewDefaultAPIRouter(optlyCache optimizely.Cache, conf *config.AgentConfig, overrideHandler = forbiddenHandler("Overrides not enabled") } - nStreamHandler := handlers.NotificationEventStreamDecider(&conf.Synchronization) + nStreamHandler := handlers.NotificationEventStreamHandler(&conf.Synchronization) if !conf.API.EnableNotifications { nStreamHandler = forbiddenHandler("Notification stream not enabled") } From 93c7a692fe4f06649aaefbdff8140cb512178942 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Tue, 12 Sep 2023 18:36:25 +0600 Subject: [PATCH 11/33] update config management --- Dockerfile | 2 +- config.yaml | 9 ++- config/config.go | 22 ++--- pkg/handlers/notification.go | 15 ++-- pkg/optimizely/cache.go | 22 ++--- pkg/syncer/syncer.go | 150 +++++++++++++++++++++-------------- 6 files changed, 132 insertions(+), 88 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b39880b..8e47d626 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,4 +11,4 @@ COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /go/src/github.com/optimizely/agent/bin/optimizely /optimizely COPY --from=builder /etc/passwd /etc/passwd USER agentuser -CMD ["/optimizely"] \ No newline at end of file +CMD ["/optimizely"] diff --git a/config.yaml b/config.yaml index 4f688e13..3f1ecdb5 100644 --- a/config.yaml +++ b/config.yaml @@ -212,7 +212,10 @@ runtime: synchronization: notification: enable: true + default: "redis" pubsub: - addr: "redis.demo.svc:6379" - password: "" - db: 0 + redis: + host: "redis.demo.svc:6379" + password: "" + database: 0 + channel: "optimizely-notifications" diff --git a/config/config.go b/config/config.go index 1f4100e2..a31ffdde 100644 --- a/config/config.go +++ b/config/config.go @@ -125,7 +125,16 @@ func NewDefaultConfig() *AgentConfig { }, Synchronization: SyncConfig{ Notification: NotificationConfig{ - Enable: false, + Enable: false, + Default: "redis", + Pubsub: map[string]interface{}{ + "redis": map[string]interface{}{ + "host": "localhost:6379", + "password": "", + "database": 0, + "channel": "optimizely-notifications", + }, + }, }, }, } @@ -156,14 +165,9 @@ type SyncConfig struct { } type NotificationConfig struct { - Enable bool `json:"enable"` - Pubsub RedisConfig `json:"pubsub"` -} - -type RedisConfig struct { - Addr string `json:"addr"` - Password string `json:"password"` - DB int `json:"db"` + Enable bool `json:"enable"` + Default string `json:"default"` + Pubsub map[string]interface{} `json:"pubsub"` } // HTTPSDisabledWarning is logged when keyfile and certfile are not provided in server configuration diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 8eef49a6..83ea7eb0 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -209,8 +209,13 @@ func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc notification.Type }{} + redisSyncer, err := syncer.NewRedisPubSubSyncer(middleware.GetLogger(r), conf) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + for _, value := range notificationsToAdd { - id, e := nc.AddHandler(value, syncer.NewRedisPubSubSyncer(middleware.GetLogger(r), conf).GetNotificationSyncer(r.Context())) + id, e := nc.AddHandler(value, redisSyncer.GetNotificationSyncer(r.Context())) if e != nil { RenderError(e, http.StatusUnprocessableEntity, w, r) return @@ -234,14 +239,14 @@ func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc }() client := redis.NewClient(&redis.Options{ - Addr: conf.Notification.Pubsub.Addr, // Redis server address - Password: conf.Notification.Pubsub.Password, // No password - DB: conf.Notification.Pubsub.DB, // Default DB + Addr: redisSyncer.Host, + Password: redisSyncer.Password, + DB: redisSyncer.Database, }) defer client.Close() // Subscribe to a Redis channel - pubsub := client.Subscribe(r.Context(), syncer.PubSubChan) + pubsub := client.Subscribe(r.Context(), redisSyncer.Channel) defer pubsub.Close() // "raw" query string option diff --git a/pkg/optimizely/cache.go b/pkg/optimizely/cache.go index 641da908..620c9408 100644 --- a/pkg/optimizely/cache.go +++ b/pkg/optimizely/cache.go @@ -38,12 +38,10 @@ import ( "github.com/optimizely/go-sdk/pkg/odp" odpEventPkg "github.com/optimizely/go-sdk/pkg/odp/event" odpSegmentPkg "github.com/optimizely/go-sdk/pkg/odp/segment" - "github.com/optimizely/go-sdk/pkg/registry" "github.com/optimizely/go-sdk/pkg/utils" odpCachePkg "github.com/optimizely/go-sdk/pkg/odp/cache" cmap "github.com/orcaman/concurrent-map" - "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) @@ -97,9 +95,6 @@ func (c *OptlyCache) Init(conf *config.AgentConfig) { } log.Warn().Msg(message) } - - nc := registry.GetNotificationCenter(sdkKey) - nc.AddHandler(notification.Track, syncer.NewRedisPubSubSyncer(&zerolog.Logger{}, &conf.Synchronization).GetNotificationSyncer(context.TODO())) } } @@ -249,14 +244,22 @@ func defaultLoader( forcedVariations := decision.NewMapExperimentOverridesStore() optimizelyFactory := &client.OptimizelyFactory{SDKKey: sdkKey} + nc := notification.NewNotificationCenter() + redisSyncer, err := syncer.NewRedisPubSubSyncer(nil, &conff.Synchronization) + if err != nil { + return nil, err + } + _, e := nc.AddHandler(notification.Track, redisSyncer.GetNotificationSyncer(context.TODO())) + if e != nil { + return nil, e + } + clientOptions := []client.OptionFunc{ client.WithConfigManager(configManager), client.WithExperimentOverrides(forcedVariations), client.WithEventProcessor(ep), client.WithOdpDisabled(conf.ODP.Disable), - } - if conff.Synchronization.Notification.Enable { - clientOptions = append(clientOptions, client.WithNotificationCenter(syncer.NewRedisCenter(&conff.Synchronization))) + client.WithNotificationCenter(nc), } var clientUserProfileService decision.UserProfileService @@ -311,9 +314,6 @@ func defaultLoader( optimizelyClient, err := optimizelyFactory.Client( clientOptions..., ) - if err != nil { - return nil, err - } return &OptlyClient{optimizelyClient, configManager, forcedVariations, clientUserProfileService, clientODPCache}, err } } diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index 3d2ebabc..7cdc29d6 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -3,10 +3,10 @@ package syncer import ( "context" "encoding/json" + "errors" "github.com/go-redis/redis/v8" "github.com/optimizely/agent/config" - "github.com/optimizely/go-sdk/pkg/notification" "github.com/rs/zerolog" ) @@ -15,71 +15,103 @@ const ( PubSubRedis = "redis" ) -type RedisCenter struct { - Addr string +// type RedisCenter struct { +// Host string +// Password string +// Database int +// Channel string +// logger *zerolog.Logger +// } + +// // AddHandler(Type, func(interface{})) (int, error) +// // RemoveHandler(int, Type) error +// // Send(Type, interface{}) error + +// func NewRedisCenter(conf *config.SyncConfig) (*RedisCenter, error) { +// return &RedisCenter{ +// Addr: conf.Notification.Pubsub.Addr, +// Password: conf.Notification.Pubsub.Password, +// DB: conf.Notification.Pubsub.DB, +// } +// } + +// func (r *RedisCenter) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { +// return 0, nil +// } + +// func (r *RedisCenter) RemoveHandler(_ int, t notification.Type) error { +// return nil +// } + +// func (r *RedisCenter) Send(_ notification.Type, n interface{}) error { +// jsonEvent, err := json.Marshal(n) +// if err != nil { +// r.logger.Error().Msg("encoding notification to json") +// return err +// } +// client := redis.NewClient(&redis.Options{ +// Addr: r.Addr, // Redis server address +// Password: r.Password, // No password +// DB: r.DB, // Default DB +// }) +// defer client.Close() + +// // Subscribe to a Redis channel +// pubsub := client.Subscribe(context.TODO(), PubSubChan) +// defer pubsub.Close() + +// if err := client.Publish(context.TODO(), PubSubChan, jsonEvent).Err(); err != nil { +// r.logger.Err(err).Msg("failed to publish json event to pub/sub") +// return err +// } +// return nil +// } + +type RedisPubSubSyncer struct { + Host string Password string - DB int + Database int + Channel string logger *zerolog.Logger } -// AddHandler(Type, func(interface{})) (int, error) -// RemoveHandler(int, Type) error -// Send(Type, interface{}) error - -func NewRedisCenter(conf *config.SyncConfig) *RedisCenter { - return &RedisCenter{ - Addr: conf.Notification.Pubsub.Addr, - Password: conf.Notification.Pubsub.Password, - DB: conf.Notification.Pubsub.DB, +func NewRedisPubSubSyncer(logger *zerolog.Logger, conf *config.SyncConfig) (*RedisPubSubSyncer, error) { + if !conf.Notification.Enable { + return nil, errors.New("notification syncer is not enabled") + } + if conf.Notification.Default != PubSubRedis { + return nil, errors.New("redis syncer is not set as default") } -} - -func (r *RedisCenter) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { - return 0, nil -} -func (r *RedisCenter) RemoveHandler(_ int, t notification.Type) error { - return nil -} + redisConfig, found := conf.Notification.Pubsub[PubSubRedis].(map[string]interface{}) + if !found { + return nil, errors.New("redis pubsub config not found") + } -func (r *RedisCenter) Send(_ notification.Type, n interface{}) error { - jsonEvent, err := json.Marshal(n) - if err != nil { - r.logger.Error().Msg("encoding notification to json") - return err + host, ok := redisConfig["host"].(string) + if !ok { + return nil, errors.New("redis host not provided in correct format") } - client := redis.NewClient(&redis.Options{ - Addr: r.Addr, // Redis server address - Password: r.Password, // No password - DB: r.DB, // Default DB - }) - defer client.Close() - - // Subscribe to a Redis channel - pubsub := client.Subscribe(context.TODO(), PubSubChan) - defer pubsub.Close() - - if err := client.Publish(context.TODO(), PubSubChan, jsonEvent).Err(); err != nil { - r.logger.Err(err).Msg("failed to publish json event to pub/sub") - return err + password, ok := redisConfig["password"].(string) + if !ok { + return nil, errors.New("redis password not provider in correct format") + } + database, ok := redisConfig["database"].(int) + if !ok { + return nil, errors.New("redis database not provided in correct format") + } + channel, ok := redisConfig["channel"].(string) + if !ok { + return nil, errors.New("redis channel not provided in correct format") } - return nil -} - -type RedisPubSubSyncer struct { - Addr string - Password string - DB int - logger *zerolog.Logger -} -func NewRedisPubSubSyncer(logger *zerolog.Logger, conf *config.SyncConfig) *RedisPubSubSyncer { return &RedisPubSubSyncer{ - Addr: conf.Notification.Pubsub.Addr, - Password: conf.Notification.Pubsub.Password, - DB: conf.Notification.Pubsub.DB, + Host: host, + Password: password, + Database: database, + Channel: channel, logger: logger, - } + }, nil } func (r *RedisPubSubSyncer) GetNotificationSyncer(ctx context.Context) func(n interface{}) { @@ -90,17 +122,17 @@ func (r *RedisPubSubSyncer) GetNotificationSyncer(ctx context.Context) func(n in return } client := redis.NewClient(&redis.Options{ - Addr: r.Addr, // Redis server address - Password: r.Password, // No password - DB: r.DB, // Default DB + Addr: r.Host, + Password: r.Password, + DB: r.Database, }) defer client.Close() // Subscribe to a Redis channel - pubsub := client.Subscribe(ctx, PubSubChan) + pubsub := client.Subscribe(ctx, r.Channel) defer pubsub.Close() - if err := client.Publish(ctx, PubSubChan, jsonEvent).Err(); err != nil { + if err := client.Publish(ctx, r.Channel, jsonEvent).Err(); err != nil { r.logger.Err(err).Msg("failed to publish json event to pub/sub") return } From a2e9f0b529699358b95c17a153c0d4707cc839a5 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Tue, 12 Sep 2023 19:11:28 +0600 Subject: [PATCH 12/33] update notidication sender --- pkg/handlers/notification.go | 45 ------------------- pkg/optimizely/cache.go | 20 ++++----- pkg/syncer/syncer.go | 85 ++++++++++++++---------------------- 3 files changed, 41 insertions(+), 109 deletions(-) diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 83ea7eb0..4d2275e1 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -183,61 +183,16 @@ func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc return } - optlyClient, err := middleware.GetOptlyClient(r) - if err != nil { - RenderError(err, http.StatusUnprocessableEntity, w, r) - return - } - - nc := optlyClient.GetNotificationCenter() - // Set the headers related to event streaming. w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") - // Parse the form. - _ = r.ParseForm() - - filters := r.Form["filter"] - - // Parse out the any filters that were added - notificationsToAdd := getFilter(filters) - - ids := []struct { - int - notification.Type - }{} - redisSyncer, err := syncer.NewRedisPubSubSyncer(middleware.GetLogger(r), conf) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } - for _, value := range notificationsToAdd { - id, e := nc.AddHandler(value, redisSyncer.GetNotificationSyncer(r.Context())) - if e != nil { - RenderError(e, http.StatusUnprocessableEntity, w, r) - return - } - - // do defer outside the loop. - ids = append(ids, struct { - int - notification.Type - }{id, value}) - } - - // Remove the decision listener if we exited. - defer func() { - for _, id := range ids { - err := nc.RemoveHandler(id.int, id.Type) - if err != nil { - middleware.GetLogger(r).Error().AnErr("removing notification", err) - } - } - }() - client := redis.NewClient(&redis.Options{ Addr: redisSyncer.Host, Password: redisSyncer.Password, diff --git a/pkg/optimizely/cache.go b/pkg/optimizely/cache.go index 620c9408..43d03fb2 100644 --- a/pkg/optimizely/cache.go +++ b/pkg/optimizely/cache.go @@ -34,7 +34,6 @@ import ( "github.com/optimizely/go-sdk/pkg/decision" "github.com/optimizely/go-sdk/pkg/event" "github.com/optimizely/go-sdk/pkg/logging" - "github.com/optimizely/go-sdk/pkg/notification" "github.com/optimizely/go-sdk/pkg/odp" odpEventPkg "github.com/optimizely/go-sdk/pkg/odp/event" odpSegmentPkg "github.com/optimizely/go-sdk/pkg/odp/segment" @@ -244,22 +243,19 @@ func defaultLoader( forcedVariations := decision.NewMapExperimentOverridesStore() optimizelyFactory := &client.OptimizelyFactory{SDKKey: sdkKey} - nc := notification.NewNotificationCenter() - redisSyncer, err := syncer.NewRedisPubSubSyncer(nil, &conff.Synchronization) - if err != nil { - return nil, err - } - _, e := nc.AddHandler(notification.Track, redisSyncer.GetNotificationSyncer(context.TODO())) - if e != nil { - return nil, e - } - clientOptions := []client.OptionFunc{ client.WithConfigManager(configManager), client.WithExperimentOverrides(forcedVariations), client.WithEventProcessor(ep), client.WithOdpDisabled(conf.ODP.Disable), - client.WithNotificationCenter(nc), + } + + if conff.Synchronization.Notification.Enable { + redisSyncer, err := syncer.NewRedisPubSubSyncer(nil, &conff.Synchronization) + if err != nil { + return nil, err + } + clientOptions = append(clientOptions, client.WithNotificationCenter(redisSyncer)) } var clientUserProfileService decision.UserProfileService diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index 7cdc29d6..3f65fd50 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -7,6 +7,7 @@ import ( "github.com/go-redis/redis/v8" "github.com/optimizely/agent/config" + "github.com/optimizely/go-sdk/pkg/notification" "github.com/rs/zerolog" ) @@ -15,58 +16,6 @@ const ( PubSubRedis = "redis" ) -// type RedisCenter struct { -// Host string -// Password string -// Database int -// Channel string -// logger *zerolog.Logger -// } - -// // AddHandler(Type, func(interface{})) (int, error) -// // RemoveHandler(int, Type) error -// // Send(Type, interface{}) error - -// func NewRedisCenter(conf *config.SyncConfig) (*RedisCenter, error) { -// return &RedisCenter{ -// Addr: conf.Notification.Pubsub.Addr, -// Password: conf.Notification.Pubsub.Password, -// DB: conf.Notification.Pubsub.DB, -// } -// } - -// func (r *RedisCenter) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { -// return 0, nil -// } - -// func (r *RedisCenter) RemoveHandler(_ int, t notification.Type) error { -// return nil -// } - -// func (r *RedisCenter) Send(_ notification.Type, n interface{}) error { -// jsonEvent, err := json.Marshal(n) -// if err != nil { -// r.logger.Error().Msg("encoding notification to json") -// return err -// } -// client := redis.NewClient(&redis.Options{ -// Addr: r.Addr, // Redis server address -// Password: r.Password, // No password -// DB: r.DB, // Default DB -// }) -// defer client.Close() - -// // Subscribe to a Redis channel -// pubsub := client.Subscribe(context.TODO(), PubSubChan) -// defer pubsub.Close() - -// if err := client.Publish(context.TODO(), PubSubChan, jsonEvent).Err(); err != nil { -// r.logger.Err(err).Msg("failed to publish json event to pub/sub") -// return err -// } -// return nil -// } - type RedisPubSubSyncer struct { Host string Password string @@ -114,6 +63,14 @@ func NewRedisPubSubSyncer(logger *zerolog.Logger, conf *config.SyncConfig) (*Red }, nil } +func (r *RedisPubSubSyncer) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { + return 0, nil +} + +func (r *RedisPubSubSyncer) RemoveHandler(_ int, t notification.Type) error { + return nil +} + func (r *RedisPubSubSyncer) GetNotificationSyncer(ctx context.Context) func(n interface{}) { return func(n interface{}) { jsonEvent, err := json.Marshal(n) @@ -138,3 +95,27 @@ func (r *RedisPubSubSyncer) GetNotificationSyncer(ctx context.Context) func(n in } } } + +func (r *RedisPubSubSyncer) Send(_ notification.Type, n interface{}) error { + jsonEvent, err := json.Marshal(n) + if err != nil { + r.logger.Error().Msg("encoding notification to json") + return err + } + client := redis.NewClient(&redis.Options{ + Addr: r.Host, // Redis server address + Password: r.Password, // No password + DB: r.Database, // Default DB + }) + defer client.Close() + + // Subscribe to a Redis channel + pubsub := client.Subscribe(context.TODO(), r.Channel) + defer pubsub.Close() + + if err := client.Publish(context.TODO(), r.Channel, jsonEvent).Err(); err != nil { + r.logger.Err(err).Msg("failed to publish json event to pub/sub") + return err + } + return nil +} From a9734689c7b5d604f647223028558f00f3e75114 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Tue, 12 Sep 2023 19:16:09 +0600 Subject: [PATCH 13/33] cleaner syncup code --- pkg/syncer/syncer.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index 3f65fd50..cca06c50 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -71,31 +71,6 @@ func (r *RedisPubSubSyncer) RemoveHandler(_ int, t notification.Type) error { return nil } -func (r *RedisPubSubSyncer) GetNotificationSyncer(ctx context.Context) func(n interface{}) { - return func(n interface{}) { - jsonEvent, err := json.Marshal(n) - if err != nil { - r.logger.Error().Msg("encoding notification to json") - return - } - client := redis.NewClient(&redis.Options{ - Addr: r.Host, - Password: r.Password, - DB: r.Database, - }) - defer client.Close() - - // Subscribe to a Redis channel - pubsub := client.Subscribe(ctx, r.Channel) - defer pubsub.Close() - - if err := client.Publish(ctx, r.Channel, jsonEvent).Err(); err != nil { - r.logger.Err(err).Msg("failed to publish json event to pub/sub") - return - } - } -} - func (r *RedisPubSubSyncer) Send(_ notification.Type, n interface{}) error { jsonEvent, err := json.Marshal(n) if err != nil { From 23c72d28ba41d19d34f987836ef9d5809cef18bb Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Tue, 12 Sep 2023 20:47:28 +0600 Subject: [PATCH 14/33] update test --- cmd/optimizely/main.go | 2 +- pkg/handlers/notification_test.go | 7 +++++-- pkg/optimizely/cache.go | 4 ++-- pkg/optimizely/cache_test.go | 24 ++++++++++++------------ pkg/routers/api_test.go | 8 ++++---- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/cmd/optimizely/main.go b/cmd/optimizely/main.go index f744bfb0..8fdf556d 100644 --- a/cmd/optimizely/main.go +++ b/cmd/optimizely/main.go @@ -151,7 +151,7 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) // Create default service context sg := server.NewGroup(ctx, conf.Server) // Create a new server group to manage the individual http listeners optlyCache := optimizely.NewCache(ctx, conf, sdkMetricsRegistry) - optlyCache.Init(conf) + optlyCache.Init(conf.SDKKeys) // goroutine to check for signals to gracefully shutdown listeners go func() { diff --git a/pkg/handlers/notification_test.go b/pkg/handlers/notification_test.go index 595c87b9..c950059d 100644 --- a/pkg/handlers/notification_test.go +++ b/pkg/handlers/notification_test.go @@ -27,6 +27,7 @@ import ( "github.com/optimizely/go-sdk/pkg/notification" "github.com/optimizely/go-sdk/pkg/registry" + "github.com/optimizely/agent/config" "github.com/optimizely/agent/pkg/middleware" "github.com/optimizely/agent/pkg/optimizely" "github.com/optimizely/agent/pkg/optimizely/optimizelytest" @@ -65,8 +66,9 @@ func (suite *NotificationTestSuite) SetupTest() { mux := chi.NewMux() EventStreamMW := &NotificationMW{optlyClient} + conf := config.NewDefaultConfig() mux.Use(EventStreamMW.ClientCtx) - mux.Get("/notifications/event-stream", NotificationEventSteamHandler) + mux.Get("/notifications/event-stream", NotificationEventStreamHandler(&conf.Synchronization)) suite.mux = mux suite.tc = testClient @@ -201,8 +203,9 @@ func TestEventStreamMissingOptlyCtx(t *testing.T) { mw := new(NotificationMW) mw.optlyClient = nil + conf := config.NewDefaultConfig() handlers := []func(w http.ResponseWriter, r *http.Request){ - NotificationEventSteamHandler, + NotificationEventStreamHandler(&conf.Synchronization), } for _, handler := range handlers { diff --git a/pkg/optimizely/cache.go b/pkg/optimizely/cache.go index 43d03fb2..ceda0835 100644 --- a/pkg/optimizely/cache.go +++ b/pkg/optimizely/cache.go @@ -84,8 +84,8 @@ func NewCache(ctx context.Context, conf *config.AgentConfig, metricsRegistry *Me } // Init takes a slice of sdkKeys to warm the cache upon startup -func (c *OptlyCache) Init(conf *config.AgentConfig) { - for _, sdkKey := range conf.SDKKeys { +func (c *OptlyCache) Init(sdkKeys []string) { + for _, sdkKey := range sdkKeys { if _, err := c.GetClient(sdkKey); err != nil { message := "Failed to initialize Optimizely Client." if ShouldIncludeSDKKey { diff --git a/pkg/optimizely/cache_test.go b/pkg/optimizely/cache_test.go index 2beabb4b..cbc97f02 100644 --- a/pkg/optimizely/cache_test.go +++ b/pkg/optimizely/cache_test.go @@ -112,7 +112,7 @@ func (suite *CacheTestSuite) TestNewCache() { sdkMetricsRegistry := NewRegistry(agentMetricsRegistry) // To improve coverage - optlyCache := NewCache(context.Background(), config.ClientConfig{}, sdkMetricsRegistry) + optlyCache := NewCache(context.Background(), &config.AgentConfig{}, sdkMetricsRegistry) suite.NotNil(optlyCache) } @@ -420,7 +420,7 @@ func (s *DefaultLoaderTestSuite) TestDefaultLoader() { }, } - loader := defaultLoader(conf, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) @@ -474,7 +474,7 @@ func (s *DefaultLoaderTestSuite) TestUPSAndODPCacheHeaderOverridesDefaultKey() { tmpOdpCacheMap := cmap.New() tmpOdpCacheMap.Set("sdkkey", "in-memory") - loader := defaultLoader(conf, s.registry, tmpUPSMap, tmpOdpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, tmpUPSMap, tmpOdpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) @@ -538,7 +538,7 @@ func (s *DefaultLoaderTestSuite) TestFirstSaveConfiguresClientForRedisUPSAndODPC }}, }, } - loader := defaultLoader(conf, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.NotNil(client.UserProfileService) @@ -596,7 +596,7 @@ func (s *DefaultLoaderTestSuite) TestFirstSaveConfiguresLRUCacheForInMemoryCache }}, }, } - loader := defaultLoader(conf, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.NotNil(client.odpCache) @@ -627,7 +627,7 @@ func (s *DefaultLoaderTestSuite) TestHttpClientInitializesByDefaultRestUPS() { "rest": map[string]interface{}{}, }}, } - loader := defaultLoader(conf, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.NotNil(client.UserProfileService) @@ -655,7 +655,7 @@ func (s *DefaultLoaderTestSuite) TestLoaderWithValidUserProfileServices() { }, }}, } - loader := defaultLoader(conf, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) @@ -686,7 +686,7 @@ func (s *DefaultLoaderTestSuite) TestLoaderWithValidODPCache() { }}, }, } - loader := defaultLoader(conf, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) @@ -709,7 +709,7 @@ func (s *DefaultLoaderTestSuite) TestLoaderWithEmptyUserProfileServices() { conf := config.ClientConfig{ UserProfileService: map[string]interface{}{}, } - loader := defaultLoader(conf, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.Nil(client.UserProfileService) @@ -726,7 +726,7 @@ func (s *DefaultLoaderTestSuite) TestLoaderWithEmptyODPCache() { SegmentsCache: map[string]interface{}{}, }, } - loader := defaultLoader(conf, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.Nil(client.odpCache) @@ -743,7 +743,7 @@ func (s *DefaultLoaderTestSuite) TestLoaderWithNoDefaultUserProfileServices() { "mock3": map[string]interface{}{}, }}, } - loader := defaultLoader(conf, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.Nil(client.UserProfileService) @@ -762,7 +762,7 @@ func (s *DefaultLoaderTestSuite) TestLoaderWithNoDefaultODPCache() { }}, }, } - loader := defaultLoader(conf, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.Nil(client.odpCache) diff --git a/pkg/routers/api_test.go b/pkg/routers/api_test.go index a34244f4..4dd81da7 100644 --- a/pkg/routers/api_test.go +++ b/pkg/routers/api_test.go @@ -331,7 +331,7 @@ func TestAPIV1TestSuite(t *testing.T) { } func TestNewDefaultAPIV1Router(t *testing.T) { - client := NewDefaultAPIRouter(MockCache{}, config.APIConfig{}, metricsRegistry) + client := NewDefaultAPIRouter(MockCache{}, &config.AgentConfig{}, metricsRegistry) assert.NotNil(t, client) } @@ -356,7 +356,7 @@ func TestNewDefaultAPIV1RouterInvalidHandlerConfig(t *testing.T) { EnableNotifications: false, EnableOverrides: false, } - client := NewDefaultAPIRouter(MockCache{}, invalidAPIConfig, metricsRegistry) + client := NewDefaultAPIRouter(MockCache{}, &config.AgentConfig{API: invalidAPIConfig}, metricsRegistry) assert.Nil(t, client) } @@ -371,12 +371,12 @@ func TestNewDefaultClientRouterInvalidMiddlewareConfig(t *testing.T) { EnableNotifications: false, EnableOverrides: false, } - client := NewDefaultAPIRouter(MockCache{}, invalidAPIConfig, metricsRegistry) + client := NewDefaultAPIRouter(MockCache{}, &config.AgentConfig{API: invalidAPIConfig}, metricsRegistry) assert.Nil(t, client) } func TestForbiddenRoutes(t *testing.T) { - conf := config.APIConfig{} + conf := &config.AgentConfig{} mux := NewDefaultAPIRouter(MockCache{}, conf, metricsRegistry) routes := []struct { From d43935a2ffc658dbb5246195015f2e9f59d2b4ec Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Tue, 12 Sep 2023 21:42:34 +0600 Subject: [PATCH 15/33] add notification filtering --- pkg/handlers/notification.go | 44 ++++++++++++++++++++++++++---------- pkg/syncer/syncer.go | 13 +++++++++-- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 4d2275e1..990d68bc 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -37,14 +37,14 @@ import ( type MessageChan chan []byte // types of notifications supported. -var types = map[string]notification.Type{ - string(notification.Decision): notification.Decision, - string(notification.Track): notification.Track, - string(notification.ProjectConfigUpdate): notification.ProjectConfigUpdate, +var types = map[notification.Type]string{ + notification.Decision: string(notification.Decision), + notification.Track: string(notification.Track), + notification.ProjectConfigUpdate: string(notification.ProjectConfigUpdate), } -func getFilter(filters []string) map[string]notification.Type { - notificationsToAdd := map[string]notification.Type{} +func getFilter(filters []string) map[notification.Type]string { + notificationsToAdd := make(map[notification.Type]string) // Parse out the any filters that were added if len(filters) == 0 { notificationsToAdd = types @@ -55,8 +55,8 @@ func getFilter(filters []string) map[string]notification.Type { splits := strings.Split(filter, ",") for _, split := range splits { // if the string is a valid type - if _, ok := types[split]; ok { - notificationsToAdd[split] = notification.Type(split) + if _, ok := types[notification.Type(split)]; ok { + notificationsToAdd[notification.Type(split)] = split } } } @@ -112,8 +112,8 @@ func NotificationEventSteamMonolithHandler(w http.ResponseWriter, r *http.Reques notification.Type }{} - for _, value := range notificationsToAdd { - id, e := nc.AddHandler(value, func(n interface{}) { + for notificationType, _ := range notificationsToAdd { + id, e := nc.AddHandler(notificationType, func(n interface{}) { jsonEvent, err := json.Marshal(n) if err != nil { middleware.GetLogger(r).Error().Msg("encoding notification to json") @@ -130,7 +130,7 @@ func NotificationEventSteamMonolithHandler(w http.ResponseWriter, r *http.Reques ids = append(ids, struct { int notification.Type - }{id, value}) + }{id, notificationType}) } // Remove the decision listener if we exited. @@ -208,6 +208,14 @@ func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc // If provided, send raw JSON lines instead of SSE-compliant strings. raw := len(r.Form["raw"]) > 0 + // Parse the form. + _ = r.ParseForm() + + filters := r.Form["filter"] + + // Parse out the any filters that were added + notificationsToAdd := getFilter(filters) + // Listen to connection close and un-register messageChan notify := r.Context().Done() @@ -220,9 +228,21 @@ func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc log.Println("looking for redis message") msg, err := pubsub.ReceiveMessage(r.Context()) if err != nil { - log.Println("Error receiving message:", err) + middleware.GetLogger(r).Err(err).Msg("error in receiving msg from redis") return } + + var notification syncer.Notification + if err := json.Unmarshal([]byte(msg.String()), ¬ification); err != nil { + middleware.GetLogger(r).Err(err).Msg("bad formatted notification") + continue + } + + _, found := notificationsToAdd[notification.Type] + if !found { + continue + } + if raw { // Raw JSON events, one per line _, _ = fmt.Fprintf(w, "%s\n", msg.Payload) diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index cca06c50..263d38ec 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -16,6 +16,11 @@ const ( PubSubRedis = "redis" ) +type Notification struct { + Type notification.Type + Message interface{} +} + type RedisPubSubSyncer struct { Host string Password string @@ -71,8 +76,12 @@ func (r *RedisPubSubSyncer) RemoveHandler(_ int, t notification.Type) error { return nil } -func (r *RedisPubSubSyncer) Send(_ notification.Type, n interface{}) error { - jsonEvent, err := json.Marshal(n) +func (r *RedisPubSubSyncer) Send(t notification.Type, n interface{}) error { + notification := Notification{ + Type: t, + Message: n, + } + jsonEvent, err := json.Marshal(notification) if err != nil { r.logger.Error().Msg("encoding notification to json") return err From 30e5c0a9ba0b43dd7410d293649b3a650b67d7ba Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Tue, 12 Sep 2023 23:10:20 +0600 Subject: [PATCH 16/33] fix bug --- pkg/handlers/notification.go | 12 +++++++++--- pkg/syncer/syncer.go | 21 +++++++++++---------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 990d68bc..60717264 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -233,7 +233,7 @@ func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc } var notification syncer.Notification - if err := json.Unmarshal([]byte(msg.String()), ¬ification); err != nil { + if err := json.Unmarshal([]byte(msg.Payload), ¬ification); err != nil { middleware.GetLogger(r).Err(err).Msg("bad formatted notification") continue } @@ -243,12 +243,18 @@ func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc continue } + jsonEvent, err := json.Marshal(notification.Message) + if err != nil { + middleware.GetLogger(r).Err(err).Msg("failed to marshal notification into json") + continue + } + if raw { // Raw JSON events, one per line - _, _ = fmt.Fprintf(w, "%s\n", msg.Payload) + _, _ = fmt.Fprintf(w, "%s\n", string(jsonEvent)) } else { // Server Sent Events compatible - _, _ = fmt.Fprintf(w, "data: %s\n\n", msg.Payload) + _, _ = fmt.Fprintf(w, "data: %s\n\n", string(jsonEvent)) } // Flush the data immediately instead of buffering it for later. // The flush will fail if the connection is closed. That will cause the handler to exit. diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index 263d38ec..ca354a95 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -17,8 +17,8 @@ const ( ) type Notification struct { - Type notification.Type - Message interface{} + Type notification.Type `json:"type"` + Message interface{} `json:"message"` } type RedisPubSubSyncer struct { @@ -59,6 +59,10 @@ func NewRedisPubSubSyncer(logger *zerolog.Logger, conf *config.SyncConfig) (*Red return nil, errors.New("redis channel not provided in correct format") } + if logger == nil { + logger = &zerolog.Logger{} + } + return &RedisPubSubSyncer{ Host: host, Password: password, @@ -81,22 +85,19 @@ func (r *RedisPubSubSyncer) Send(t notification.Type, n interface{}) error { Type: t, Message: n, } + jsonEvent, err := json.Marshal(notification) if err != nil { - r.logger.Error().Msg("encoding notification to json") return err } + client := redis.NewClient(&redis.Options{ - Addr: r.Host, // Redis server address - Password: r.Password, // No password - DB: r.Database, // Default DB + Addr: r.Host, + Password: r.Password, + DB: r.Database, }) defer client.Close() - // Subscribe to a Redis channel - pubsub := client.Subscribe(context.TODO(), r.Channel) - defer pubsub.Close() - if err := client.Publish(context.TODO(), r.Channel, jsonEvent).Err(); err != nil { r.logger.Err(err).Msg("failed to publish json event to pub/sub") return err From f6fad826f3de91a86307ff605dee98065cd4f272 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Wed, 13 Sep 2023 17:06:31 +0600 Subject: [PATCH 17/33] rename syncer struct --- pkg/handlers/notification.go | 2 +- pkg/optimizely/cache.go | 2 +- pkg/syncer/syncer.go | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 60717264..3bd9ee39 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -188,7 +188,7 @@ func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") - redisSyncer, err := syncer.NewRedisPubSubSyncer(middleware.GetLogger(r), conf) + redisSyncer, err := syncer.NewRedisNotificationSyncer(middleware.GetLogger(r), conf) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } diff --git a/pkg/optimizely/cache.go b/pkg/optimizely/cache.go index ceda0835..cf4f3701 100644 --- a/pkg/optimizely/cache.go +++ b/pkg/optimizely/cache.go @@ -251,7 +251,7 @@ func defaultLoader( } if conff.Synchronization.Notification.Enable { - redisSyncer, err := syncer.NewRedisPubSubSyncer(nil, &conff.Synchronization) + redisSyncer, err := syncer.NewRedisNotificationSyncer(nil, &conff.Synchronization) if err != nil { return nil, err } diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index ca354a95..8e9a3ee4 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -21,7 +21,7 @@ type Notification struct { Message interface{} `json:"message"` } -type RedisPubSubSyncer struct { +type RedisNotificationSyncer struct { Host string Password string Database int @@ -29,7 +29,7 @@ type RedisPubSubSyncer struct { logger *zerolog.Logger } -func NewRedisPubSubSyncer(logger *zerolog.Logger, conf *config.SyncConfig) (*RedisPubSubSyncer, error) { +func NewRedisNotificationSyncer(logger *zerolog.Logger, conf *config.SyncConfig) (*RedisNotificationSyncer, error) { if !conf.Notification.Enable { return nil, errors.New("notification syncer is not enabled") } @@ -63,7 +63,7 @@ func NewRedisPubSubSyncer(logger *zerolog.Logger, conf *config.SyncConfig) (*Red logger = &zerolog.Logger{} } - return &RedisPubSubSyncer{ + return &RedisNotificationSyncer{ Host: host, Password: password, Database: database, @@ -72,15 +72,15 @@ func NewRedisPubSubSyncer(logger *zerolog.Logger, conf *config.SyncConfig) (*Red }, nil } -func (r *RedisPubSubSyncer) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { +func (r *RedisNotificationSyncer) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { return 0, nil } -func (r *RedisPubSubSyncer) RemoveHandler(_ int, t notification.Type) error { +func (r *RedisNotificationSyncer) RemoveHandler(_ int, t notification.Type) error { return nil } -func (r *RedisPubSubSyncer) Send(t notification.Type, n interface{}) error { +func (r *RedisNotificationSyncer) Send(t notification.Type, n interface{}) error { notification := Notification{ Type: t, Message: n, From fe55787f5d5a7a712bd323837cb43e1ed1bafaf6 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Wed, 13 Sep 2023 22:33:34 +0600 Subject: [PATCH 18/33] not using pointers --- cmd/optimizely/main.go | 2 +- pkg/handlers/notification.go | 4 ++-- pkg/optimizely/cache.go | 45 ++++++++++++++++++------------------ pkg/routers/api.go | 2 +- pkg/syncer/syncer.go | 2 +- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/cmd/optimizely/main.go b/cmd/optimizely/main.go index 8fdf556d..78db3ee6 100644 --- a/cmd/optimizely/main.go +++ b/cmd/optimizely/main.go @@ -150,7 +150,7 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) // Create default service context sg := server.NewGroup(ctx, conf.Server) // Create a new server group to manage the individual http listeners - optlyCache := optimizely.NewCache(ctx, conf, sdkMetricsRegistry) + optlyCache := optimizely.NewCache(ctx, *conf, sdkMetricsRegistry) optlyCache.Init(conf.SDKKeys) // goroutine to check for signals to gracefully shutdown listeners diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 3bd9ee39..dbfc91ce 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -64,7 +64,7 @@ func getFilter(filters []string) map[notification.Type]string { return notificationsToAdd } -func NotificationEventStreamHandler(syncConfig *config.SyncConfig) http.HandlerFunc { +func NotificationEventStreamHandler(syncConfig config.SyncConfig) http.HandlerFunc { if !syncConfig.Notification.Enable { return NotificationEventSteamMonolithHandler } @@ -173,7 +173,7 @@ func NotificationEventSteamMonolithHandler(w http.ResponseWriter, r *http.Reques } // NotificationEventSteamHandler implements the http.Handler interface. -func NotificationEventSteamSyncHandler(conf *config.SyncConfig) http.HandlerFunc { +func NotificationEventSteamSyncHandler(conf config.SyncConfig) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // Make sure that the writer supports flushing. flusher, ok := w.(http.Flusher) diff --git a/pkg/optimizely/cache.go b/pkg/optimizely/cache.go index cf4f3701..43ae35ad 100644 --- a/pkg/optimizely/cache.go +++ b/pkg/optimizely/cache.go @@ -41,6 +41,7 @@ import ( odpCachePkg "github.com/optimizely/go-sdk/pkg/odp/cache" cmap "github.com/orcaman/concurrent-map" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) @@ -62,7 +63,7 @@ type OptlyCache struct { } // NewCache returns a new implementation of OptlyCache interface backed by a concurrent map. -func NewCache(ctx context.Context, conf *config.AgentConfig, metricsRegistry *MetricsRegistry) *OptlyCache { +func NewCache(ctx context.Context, conf config.AgentConfig, metricsRegistry *MetricsRegistry) *OptlyCache { // TODO is there a cleaner way to handle this translation??? cmLoader := func(sdkkey string, options ...sdkconfig.OptionFunc) SyncedConfigManager { @@ -169,14 +170,14 @@ func regexValidator(sdkKeyRegex string) func(string) bool { } func defaultLoader( - conff *config.AgentConfig, + agentConf config.AgentConfig, metricsRegistry *MetricsRegistry, userProfileServiceMap cmap.ConcurrentMap, odpCacheMap cmap.ConcurrentMap, pcFactory func(sdkKey string, options ...sdkconfig.OptionFunc) SyncedConfigManager, bpFactory func(options ...event.BPOptionConfig) *event.BatchEventProcessor) func(clientKey string) (*OptlyClient, error) { - conf := conff.Client - validator := regexValidator(conf.SdkKeyRegex) + clientConf := agentConf.Client + validator := regexValidator(clientConf.SdkKeyRegex) return func(clientKey string) (*OptlyClient, error) { var sdkKey string @@ -213,15 +214,15 @@ func defaultLoader( if datafileAccessToken != "" { configManager = pcFactory( sdkKey, - sdkconfig.WithPollingInterval(conf.PollingInterval), - sdkconfig.WithDatafileURLTemplate(conf.DatafileURLTemplate), + sdkconfig.WithPollingInterval(clientConf.PollingInterval), + sdkconfig.WithDatafileURLTemplate(clientConf.DatafileURLTemplate), sdkconfig.WithDatafileAccessToken(datafileAccessToken), ) } else { configManager = pcFactory( sdkKey, - sdkconfig.WithPollingInterval(conf.PollingInterval), - sdkconfig.WithDatafileURLTemplate(conf.DatafileURLTemplate), + sdkconfig.WithPollingInterval(clientConf.PollingInterval), + sdkconfig.WithDatafileURLTemplate(clientConf.DatafileURLTemplate), ) } @@ -229,13 +230,13 @@ func defaultLoader( return &OptlyClient{}, err } - q := event.NewInMemoryQueue(conf.QueueSize) + q := event.NewInMemoryQueue(clientConf.QueueSize) ep := bpFactory( event.WithSDKKey(sdkKey), - event.WithQueueSize(conf.QueueSize), - event.WithBatchSize(conf.BatchSize), - event.WithEventEndPoint(conf.EventURL), - event.WithFlushInterval(conf.FlushInterval), + event.WithQueueSize(clientConf.QueueSize), + event.WithBatchSize(clientConf.BatchSize), + event.WithEventEndPoint(clientConf.EventURL), + event.WithFlushInterval(clientConf.FlushInterval), event.WithQueue(q), event.WithEventDispatcherMetrics(metricsRegistry), ) @@ -247,11 +248,11 @@ func defaultLoader( client.WithConfigManager(configManager), client.WithExperimentOverrides(forcedVariations), client.WithEventProcessor(ep), - client.WithOdpDisabled(conf.ODP.Disable), + client.WithOdpDisabled(clientConf.ODP.Disable), } - if conff.Synchronization.Notification.Enable { - redisSyncer, err := syncer.NewRedisNotificationSyncer(nil, &conff.Synchronization) + if agentConf.Synchronization.Notification.Enable { + redisSyncer, err := syncer.NewRedisNotificationSyncer(&zerolog.Logger{}, agentConf.Synchronization) if err != nil { return nil, err } @@ -259,7 +260,7 @@ func defaultLoader( } var clientUserProfileService decision.UserProfileService - var rawUPS = getServiceWithType(userProfileServicePlugin, sdkKey, userProfileServiceMap, conf.UserProfileService) + var rawUPS = getServiceWithType(userProfileServicePlugin, sdkKey, userProfileServiceMap, clientConf.UserProfileService) // Check if ups was provided by user if rawUPS != nil { // convert ups to UserProfileService interface @@ -270,7 +271,7 @@ func defaultLoader( } var clientODPCache odpCachePkg.Cache - var rawODPCache = getServiceWithType(odpCachePlugin, sdkKey, odpCacheMap, conf.ODP.SegmentsCache) + var rawODPCache = getServiceWithType(odpCachePlugin, sdkKey, odpCacheMap, clientConf.ODP.SegmentsCache) // Check if odp cache was provided by user if rawODPCache != nil { // convert odpCache to Cache interface @@ -283,7 +284,7 @@ func defaultLoader( segmentManager := odpSegmentPkg.NewSegmentManager( sdkKey, odpSegmentPkg.WithAPIManager( - odpSegmentPkg.NewSegmentAPIManager(sdkKey, utils.NewHTTPRequester(logging.GetLogger(sdkKey, "SegmentAPIManager"), utils.Timeout(conf.ODP.SegmentsRequestTimeout))), + odpSegmentPkg.NewSegmentAPIManager(sdkKey, utils.NewHTTPRequester(logging.GetLogger(sdkKey, "SegmentAPIManager"), utils.Timeout(clientConf.ODP.SegmentsRequestTimeout))), ), odpSegmentPkg.WithSegmentsCache(clientODPCache), ) @@ -292,16 +293,16 @@ func defaultLoader( eventManager := odpEventPkg.NewBatchEventManager( odpEventPkg.WithAPIManager( odpEventPkg.NewEventAPIManager( - sdkKey, utils.NewHTTPRequester(logging.GetLogger(sdkKey, "EventAPIManager"), utils.Timeout(conf.ODP.EventsRequestTimeout)), + sdkKey, utils.NewHTTPRequester(logging.GetLogger(sdkKey, "EventAPIManager"), utils.Timeout(clientConf.ODP.EventsRequestTimeout)), ), ), - odpEventPkg.WithFlushInterval(conf.ODP.EventsFlushInterval), + odpEventPkg.WithFlushInterval(clientConf.ODP.EventsFlushInterval), ) // Create odp manager with custom segment and event manager odpManager := odp.NewOdpManager( sdkKey, - conf.ODP.Disable, + clientConf.ODP.Disable, odp.WithSegmentManager(segmentManager), odp.WithEventManager(eventManager), ) diff --git a/pkg/routers/api.go b/pkg/routers/api.go index 759072bf..cb91ab97 100644 --- a/pkg/routers/api.go +++ b/pkg/routers/api.go @@ -82,7 +82,7 @@ func NewDefaultAPIRouter(optlyCache optimizely.Cache, conf *config.AgentConfig, overrideHandler = forbiddenHandler("Overrides not enabled") } - nStreamHandler := handlers.NotificationEventStreamHandler(&conf.Synchronization) + nStreamHandler := handlers.NotificationEventStreamHandler(conf.Synchronization) if !conf.API.EnableNotifications { nStreamHandler = forbiddenHandler("Notification stream not enabled") } diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index 8e9a3ee4..827c944e 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -29,7 +29,7 @@ type RedisNotificationSyncer struct { logger *zerolog.Logger } -func NewRedisNotificationSyncer(logger *zerolog.Logger, conf *config.SyncConfig) (*RedisNotificationSyncer, error) { +func NewRedisNotificationSyncer(logger *zerolog.Logger, conf config.SyncConfig) (*RedisNotificationSyncer, error) { if !conf.Notification.Enable { return nil, errors.New("notification syncer is not enabled") } From fd633894f9a05b022d5d3016f261fca1976560a3 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Wed, 13 Sep 2023 23:05:16 +0600 Subject: [PATCH 19/33] fix test --- pkg/handlers/notification_test.go | 4 ++-- pkg/optimizely/cache_test.go | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/handlers/notification_test.go b/pkg/handlers/notification_test.go index c950059d..7a0ec74c 100644 --- a/pkg/handlers/notification_test.go +++ b/pkg/handlers/notification_test.go @@ -68,7 +68,7 @@ func (suite *NotificationTestSuite) SetupTest() { conf := config.NewDefaultConfig() mux.Use(EventStreamMW.ClientCtx) - mux.Get("/notifications/event-stream", NotificationEventStreamHandler(&conf.Synchronization)) + mux.Get("/notifications/event-stream", NotificationEventStreamHandler(conf.Synchronization)) suite.mux = mux suite.tc = testClient @@ -205,7 +205,7 @@ func TestEventStreamMissingOptlyCtx(t *testing.T) { conf := config.NewDefaultConfig() handlers := []func(w http.ResponseWriter, r *http.Request){ - NotificationEventStreamHandler(&conf.Synchronization), + NotificationEventStreamHandler(conf.Synchronization), } for _, handler := range handlers { diff --git a/pkg/optimizely/cache_test.go b/pkg/optimizely/cache_test.go index cbc97f02..bd8678b8 100644 --- a/pkg/optimizely/cache_test.go +++ b/pkg/optimizely/cache_test.go @@ -112,7 +112,7 @@ func (suite *CacheTestSuite) TestNewCache() { sdkMetricsRegistry := NewRegistry(agentMetricsRegistry) // To improve coverage - optlyCache := NewCache(context.Background(), &config.AgentConfig{}, sdkMetricsRegistry) + optlyCache := NewCache(context.Background(), config.AgentConfig{}, sdkMetricsRegistry) suite.NotNil(optlyCache) } @@ -420,7 +420,7 @@ func (s *DefaultLoaderTestSuite) TestDefaultLoader() { }, } - loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) @@ -474,7 +474,7 @@ func (s *DefaultLoaderTestSuite) TestUPSAndODPCacheHeaderOverridesDefaultKey() { tmpOdpCacheMap := cmap.New() tmpOdpCacheMap.Set("sdkkey", "in-memory") - loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, tmpUPSMap, tmpOdpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(config.AgentConfig{Client: conf}, s.registry, tmpUPSMap, tmpOdpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) @@ -538,7 +538,7 @@ func (s *DefaultLoaderTestSuite) TestFirstSaveConfiguresClientForRedisUPSAndODPC }}, }, } - loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.NotNil(client.UserProfileService) @@ -596,7 +596,7 @@ func (s *DefaultLoaderTestSuite) TestFirstSaveConfiguresLRUCacheForInMemoryCache }}, }, } - loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.NotNil(client.odpCache) @@ -627,7 +627,7 @@ func (s *DefaultLoaderTestSuite) TestHttpClientInitializesByDefaultRestUPS() { "rest": map[string]interface{}{}, }}, } - loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.NotNil(client.UserProfileService) @@ -655,7 +655,7 @@ func (s *DefaultLoaderTestSuite) TestLoaderWithValidUserProfileServices() { }, }}, } - loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) @@ -686,7 +686,7 @@ func (s *DefaultLoaderTestSuite) TestLoaderWithValidODPCache() { }}, }, } - loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) @@ -709,7 +709,7 @@ func (s *DefaultLoaderTestSuite) TestLoaderWithEmptyUserProfileServices() { conf := config.ClientConfig{ UserProfileService: map[string]interface{}{}, } - loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.Nil(client.UserProfileService) @@ -726,7 +726,7 @@ func (s *DefaultLoaderTestSuite) TestLoaderWithEmptyODPCache() { SegmentsCache: map[string]interface{}{}, }, } - loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.Nil(client.odpCache) @@ -743,7 +743,7 @@ func (s *DefaultLoaderTestSuite) TestLoaderWithNoDefaultUserProfileServices() { "mock3": map[string]interface{}{}, }}, } - loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.Nil(client.UserProfileService) @@ -762,7 +762,7 @@ func (s *DefaultLoaderTestSuite) TestLoaderWithNoDefaultODPCache() { }}, }, } - loader := defaultLoader(&config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) + loader := defaultLoader(config.AgentConfig{Client: conf}, s.registry, s.upsMap, s.odpCacheMap, s.pcFactory, s.bpFactory) client, err := loader("sdkkey") s.NoError(err) s.Nil(client.odpCache) From 17e9dd8b8f6b9f0be7564084842d3f0f5c0b9b74 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Thu, 14 Sep 2023 01:08:33 +0600 Subject: [PATCH 20/33] implement data channel --- config.yaml | 2 +- pkg/handlers/notification.go | 254 +++++++++++++++++++++++++----- pkg/handlers/notification_test.go | 61 ++++++- pkg/routers/api.go | 9 +- pkg/syncer/syncer.go | 9 +- 5 files changed, 287 insertions(+), 48 deletions(-) diff --git a/config.yaml b/config.yaml index 3f1ecdb5..1d89fa41 100644 --- a/config.yaml +++ b/config.yaml @@ -211,7 +211,7 @@ runtime: ## will get the notifications from multiple replicas synchronization: notification: - enable: true + enable: false default: "redis" pubsub: redis: diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index dbfc91ce..f27a1ce4 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -18,9 +18,10 @@ package handlers import ( + "context" "encoding/json" + "errors" "fmt" - "log" "net/http" "strings" @@ -30,12 +31,20 @@ import ( "github.com/optimizely/agent/pkg/syncer" "github.com/optimizely/go-sdk/pkg/notification" "github.com/optimizely/go-sdk/pkg/registry" + "github.com/rs/zerolog" +) + +const ( + LoggerKey = "notification-logger" + SDKKey = "context-sdk-key" ) // A MessageChan is a channel of bytes // Each http handler call creates a new channel and pumps decision service messages onto it. type MessageChan chan []byte +type NotificationReceiverFunc func(context.Context, config.SyncConfig) (<-chan syncer.Notification, error) + // types of notifications supported. var types = map[notification.Type]string{ notification.Decision: string(notification.Decision), @@ -64,15 +73,94 @@ func getFilter(filters []string) map[notification.Type]string { return notificationsToAdd } -func NotificationEventStreamHandler(syncConfig config.SyncConfig) http.HandlerFunc { - if !syncConfig.Notification.Enable { - return NotificationEventSteamMonolithHandler +// func NotificationEventStreamHandler(syncConfig config.SyncConfig) http.HandlerFunc { +// if syncConfig.Notification.Enable { +// return notificationEventSteamSyncHandler(syncConfig, redisNotificationReceiver) +// } +// return notificationEventSteamMonolithHandler +// } + +func GetNotificationReceiverFunc(conf config.SyncConfig) NotificationReceiverFunc { + if !conf.Notification.Enable { + return DefaultNotificationReceiver + } + return RedisNotificationReceiver +} + +func NotificationEventStreamHandler(conf config.SyncConfig, notificationReceiverFn NotificationReceiverFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Make sure that the writer supports flushing. + flusher, ok := w.(http.Flusher) + + if !ok { + http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) + return + } + + // Set the headers related to event streaming. + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + + // "raw" query string option + // If provided, send raw JSON lines instead of SSE-compliant strings. + raw := len(r.Form["raw"]) > 0 + + // Parse the form. + _ = r.ParseForm() + + filters := r.Form["filter"] + + // Parse out the any filters that were added + notificationsToAdd := getFilter(filters) + + // Listen to connection close and un-register messageChan + notify := r.Context().Done() + + sdkKey := r.Header.Get(middleware.OptlySDKHeader) + ctx := context.WithValue(r.Context(), SDKKey, sdkKey) + + dataChan, err := notificationReceiverFn(context.WithValue(ctx, LoggerKey, middleware.GetLogger(r)), conf) + if err != nil { + middleware.GetLogger(r).Err(err).Msg("error from receiver") + http.Error(w, "Error from data receiver!", http.StatusInternalServerError) + return + } + + for { + select { + case <-notify: + middleware.GetLogger(r).Debug().Msg("received close on the request. So, we are shutting down this handler") + return + case event := <-dataChan: + _, found := notificationsToAdd[event.Type] + if !found { + continue + } + + jsonEvent, err := json.Marshal(event) + if err != nil { + middleware.GetLogger(r).Err(err).Msg("failed to marshal notification into json") + continue + } + + if raw { + // Raw JSON events, one per line + _, _ = fmt.Fprintf(w, "%s\n", string(jsonEvent)) + } else { + // Server Sent Events compatible + _, _ = fmt.Fprintf(w, "data: %s\n\n", string(jsonEvent)) + } + // Flush the data immediately instead of buffering it for later. + // The flush will fail if the connection is closed. That will cause the handler to exit. + flusher.Flush() + } + } } - return NotificationEventSteamSyncHandler(syncConfig) } // NotificationEventSteamHandler implements the http.Handler interface. -func NotificationEventSteamMonolithHandler(w http.ResponseWriter, r *http.Request) { +func notificationEventSteamMonolithHandler(w http.ResponseWriter, r *http.Request) { // Make sure that the writer supports flushing. flusher, ok := w.(http.Flusher) @@ -172,8 +260,118 @@ func NotificationEventSteamMonolithHandler(w http.ResponseWriter, r *http.Reques } +func DefaultNotificationReceiver(ctx context.Context, conf config.SyncConfig) (<-chan syncer.Notification, error) { + logger, ok := ctx.Value(LoggerKey).(*zerolog.Logger) + if !ok { + logger = &zerolog.Logger{} + } + + sdkKey, ok := ctx.Value(SDKKey).(string) + if !ok || sdkKey == "" { + return nil, errors.New("sdk key not found") + } + + // Each connection registers its own message channel with the NotificationHandler's connections registry + messageChan := make(chan syncer.Notification) + nc := registry.GetNotificationCenter(sdkKey) + + // Parse out the any filters that were added + notificationsToAdd := types + + ids := []struct { + int + notification.Type + }{} + + for notificationType := range notificationsToAdd { + id, e := nc.AddHandler(notificationType, func(n interface{}) { + msg := syncer.Notification{ + Type: notificationType, + Message: n, + } + messageChan <- msg + }) + if e != nil { + return nil, e + } + + // do defer outside the loop. + ids = append(ids, struct { + int + notification.Type + }{id, notificationType}) + } + + go func() { + for { + select { + case <-ctx.Done(): + for _, id := range ids { + err := nc.RemoveHandler(id.int, id.Type) + if err != nil { + logger.Err(err).AnErr("removing notification", err) + } + } + return + } + } + }() + + return messageChan, nil +} + +func RedisNotificationReceiver(ctx context.Context, conf config.SyncConfig) (<-chan syncer.Notification, error) { + redisSyncer, err := syncer.NewRedisNotificationSyncer(&zerolog.Logger{}, conf) + if err != nil { + return nil, err + } + + client := redis.NewClient(&redis.Options{ + Addr: redisSyncer.Host, + Password: redisSyncer.Password, + DB: redisSyncer.Database, + }) + + // Subscribe to a Redis channel + pubsub := client.Subscribe(ctx, redisSyncer.Channel) + + dataChan := make(chan syncer.Notification) + + logger, ok := ctx.Value(LoggerKey).(*zerolog.Logger) + if !ok { + logger = &zerolog.Logger{} + } + + go func() { + for { + select { + case <-ctx.Done(): + client.Close() + pubsub.Close() + logger.Debug().Msg("context cancelled, redis notification receiver is closed") + return + default: + msg, err := pubsub.ReceiveMessage(ctx) + if err != nil { + logger.Err(err).Msg("failed to receive message from redis") + continue + } + + var event syncer.Notification + if err := json.Unmarshal([]byte(msg.Payload), &event); err != nil { + logger.Err(err).Msg("failed to unmarshal redis message") + continue + } + dataChan <- event + } + } + }() + + return dataChan, nil +} + // NotificationEventSteamHandler implements the http.Handler interface. -func NotificationEventSteamSyncHandler(conf config.SyncConfig) http.HandlerFunc { +func notificationEventSteamSyncHandler(conf config.SyncConfig, receiverFn NotificationReceiverFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // Make sure that the writer supports flushing. flusher, ok := w.(http.Flusher) @@ -188,22 +386,6 @@ func NotificationEventSteamSyncHandler(conf config.SyncConfig) http.HandlerFunc w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") - redisSyncer, err := syncer.NewRedisNotificationSyncer(middleware.GetLogger(r), conf) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - client := redis.NewClient(&redis.Options{ - Addr: redisSyncer.Host, - Password: redisSyncer.Password, - DB: redisSyncer.Database, - }) - defer client.Close() - - // Subscribe to a Redis channel - pubsub := client.Subscribe(r.Context(), redisSyncer.Channel) - defer pubsub.Close() - // "raw" query string option // If provided, send raw JSON lines instead of SSE-compliant strings. raw := len(r.Form["raw"]) > 0 @@ -219,31 +401,25 @@ func NotificationEventSteamSyncHandler(conf config.SyncConfig) http.HandlerFunc // Listen to connection close and un-register messageChan notify := r.Context().Done() + dataChan, err := receiverFn(context.WithValue(r.Context(), LoggerKey, middleware.GetLogger(r)), conf) + if err != nil { + middleware.GetLogger(r).Err(err).Msg("error from receiver") + http.Error(w, "Error from data receiver!", http.StatusInternalServerError) + return + } + for { select { case <-notify: middleware.GetLogger(r).Debug().Msg("received close on the request. So, we are shutting down this handler") return - default: - log.Println("looking for redis message") - msg, err := pubsub.ReceiveMessage(r.Context()) - if err != nil { - middleware.GetLogger(r).Err(err).Msg("error in receiving msg from redis") - return - } - - var notification syncer.Notification - if err := json.Unmarshal([]byte(msg.Payload), ¬ification); err != nil { - middleware.GetLogger(r).Err(err).Msg("bad formatted notification") - continue - } - - _, found := notificationsToAdd[notification.Type] + case event := <-dataChan: + _, found := notificationsToAdd[event.Type] if !found { continue } - jsonEvent, err := json.Marshal(notification.Message) + jsonEvent, err := json.Marshal(event) if err != nil { middleware.GetLogger(r).Err(err).Msg("failed to marshal notification into json") continue diff --git a/pkg/handlers/notification_test.go b/pkg/handlers/notification_test.go index 7a0ec74c..baa71eaa 100644 --- a/pkg/handlers/notification_test.go +++ b/pkg/handlers/notification_test.go @@ -66,15 +66,16 @@ func (suite *NotificationTestSuite) SetupTest() { mux := chi.NewMux() EventStreamMW := &NotificationMW{optlyClient} - conf := config.NewDefaultConfig() mux.Use(EventStreamMW.ClientCtx) - mux.Get("/notifications/event-stream", NotificationEventStreamHandler(conf.Synchronization)) suite.mux = mux suite.tc = testClient } func (suite *NotificationTestSuite) TestFeatureTestFilter() { + conf := config.NewDefaultConfig() + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(conf.Synchronization)) + feature := entities.Feature{Key: "one"} suite.tc.AddFeatureTest(feature) @@ -125,6 +126,59 @@ func (suite *NotificationTestSuite) TestFilter() { } func (suite *NotificationTestSuite) TestTrackAndProjectConfig() { + conf := config.NewDefaultConfig() + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(conf.Synchronization)) + + event := entities.Event{Key: "one"} + suite.tc.AddEvent(event) + + req := httptest.NewRequest("GET", "/notifications/event-stream", nil) + rec := httptest.NewRecorder() + + expected := `data: {"test":"value"}` + "\n\n" + `data: {"Type":"project_config_update","Revision":"revision"}` + "\n\n" + + // create a cancelable request context + ctx := req.Context() + ctx1, _ := context.WithTimeout(ctx, 3*time.Second) + + nc := registry.GetNotificationCenter("") + + go func() { + time.Sleep(1 * time.Second) + _ = nc.Send(notification.Track, map[string]string{"test": "value"}) + projectConfigUpdateNotification := notification.ProjectConfigUpdateNotification{ + Type: notification.ProjectConfigUpdate, + Revision: suite.tc.ProjectConfig.GetRevision(), + } + _ = nc.Send(notification.ProjectConfigUpdate, projectConfigUpdateNotification) + }() + + suite.mux.ServeHTTP(rec, req.WithContext(ctx1)) + + suite.Equal(http.StatusOK, rec.Code) + + // Unmarshal response + response := string(rec.Body.Bytes()) + suite.Equal(expected, response) +} + +func (suite *NotificationTestSuite) TestTrackAndProjectConfigWithSynchronization() { + conf := config.NewDefaultConfig() + conf.Synchronization = config.SyncConfig{ + Notification: config.NotificationConfig{ + Enable: true, + Default: "redis", + Pubsub: map[string]interface{}{ + "redis": map[string]interface{}{ + "host": "localhost:6379", + "password": "", + "database": 0, + }, + }, + }, + } + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(conf.Synchronization)) + event := entities.Event{Key: "one"} suite.tc.AddEvent(event) @@ -159,6 +213,9 @@ func (suite *NotificationTestSuite) TestTrackAndProjectConfig() { } func (suite *NotificationTestSuite) TestActivateExperimentRaw() { + conf := config.NewDefaultConfig() + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(conf.Synchronization)) + testVariation := suite.tc.ProjectConfig.CreateVariation("variation_a") suite.tc.AddExperiment("one", []entities.Variation{testVariation}) diff --git a/pkg/routers/api.go b/pkg/routers/api.go index cb91ab97..0c8886ef 100644 --- a/pkg/routers/api.go +++ b/pkg/routers/api.go @@ -82,9 +82,12 @@ func NewDefaultAPIRouter(optlyCache optimizely.Cache, conf *config.AgentConfig, overrideHandler = forbiddenHandler("Overrides not enabled") } - nStreamHandler := handlers.NotificationEventStreamHandler(conf.Synchronization) - if !conf.API.EnableNotifications { - nStreamHandler = forbiddenHandler("Notification stream not enabled") + nStreamHandler := forbiddenHandler("Notification stream not enabled") + if conf.API.EnableNotifications { + nStreamHandler = handlers.NotificationEventStreamHandler(conf.Synchronization, handlers.DefaultNotificationReceiver) + if conf.Synchronization.Notification.Enable { + nStreamHandler = handlers.NotificationEventStreamHandler(conf.Synchronization, handlers.RedisNotificationReceiver) + } } mw := middleware.CachedOptlyMiddleware{Cache: optlyCache} diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index 827c944e..807416f1 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -12,8 +12,8 @@ import ( ) const ( - PubSubChan = "optimizely-notifications" - PubSubRedis = "redis" + PubSubDefaultChan = "optimizely-notifications" + PubSubRedis = "redis" ) type Notification struct { @@ -36,6 +36,9 @@ func NewRedisNotificationSyncer(logger *zerolog.Logger, conf config.SyncConfig) if conf.Notification.Default != PubSubRedis { return nil, errors.New("redis syncer is not set as default") } + if conf.Notification.Pubsub == nil { + return nil, errors.New("redis config is not given") + } redisConfig, found := conf.Notification.Pubsub[PubSubRedis].(map[string]interface{}) if !found { @@ -56,7 +59,7 @@ func NewRedisNotificationSyncer(logger *zerolog.Logger, conf config.SyncConfig) } channel, ok := redisConfig["channel"].(string) if !ok { - return nil, errors.New("redis channel not provided in correct format") + channel = PubSubDefaultChan } if logger == nil { From 7e1d356c40e7ab74223c5b6f75995714b58d7f33 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Thu, 14 Sep 2023 21:29:50 +0600 Subject: [PATCH 21/33] update tests --- pkg/handlers/notification.go | 273 ++++++------------------------ pkg/handlers/notification_test.go | 107 ++++++++---- pkg/routers/api.go | 4 +- 3 files changed, 126 insertions(+), 258 deletions(-) diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index f27a1ce4..899b0f77 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -43,7 +43,7 @@ const ( // Each http handler call creates a new channel and pumps decision service messages onto it. type MessageChan chan []byte -type NotificationReceiverFunc func(context.Context, config.SyncConfig) (<-chan syncer.Notification, error) +type NotificationReceiverFunc func(context.Context) (<-chan syncer.Notification, error) // types of notifications supported. var types = map[notification.Type]string{ @@ -80,14 +80,7 @@ func getFilter(filters []string) map[notification.Type]string { // return notificationEventSteamMonolithHandler // } -func GetNotificationReceiverFunc(conf config.SyncConfig) NotificationReceiverFunc { - if !conf.Notification.Enable { - return DefaultNotificationReceiver - } - return RedisNotificationReceiver -} - -func NotificationEventStreamHandler(conf config.SyncConfig, notificationReceiverFn NotificationReceiverFunc) http.HandlerFunc { +func NotificationEventStreamHandler(notificationReceiverFn NotificationReceiverFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // Make sure that the writer supports flushing. flusher, ok := w.(http.Flusher) @@ -97,6 +90,13 @@ func NotificationEventStreamHandler(conf config.SyncConfig, notificationReceiver return } + _, err := middleware.GetOptlyClient(r) + + if err != nil { + RenderError(err, http.StatusUnprocessableEntity, w, r) + return + } + // Set the headers related to event streaming. w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") @@ -104,7 +104,7 @@ func NotificationEventStreamHandler(conf config.SyncConfig, notificationReceiver // "raw" query string option // If provided, send raw JSON lines instead of SSE-compliant strings. - raw := len(r.Form["raw"]) > 0 + raw := len(r.URL.Query()["raw"]) > 0 // Parse the form. _ = r.ParseForm() @@ -120,7 +120,7 @@ func NotificationEventStreamHandler(conf config.SyncConfig, notificationReceiver sdkKey := r.Header.Get(middleware.OptlySDKHeader) ctx := context.WithValue(r.Context(), SDKKey, sdkKey) - dataChan, err := notificationReceiverFn(context.WithValue(ctx, LoggerKey, middleware.GetLogger(r)), conf) + dataChan, err := notificationReceiverFn(context.WithValue(ctx, LoggerKey, middleware.GetLogger(r))) if err != nil { middleware.GetLogger(r).Err(err).Msg("error from receiver") http.Error(w, "Error from data receiver!", http.StatusInternalServerError) @@ -138,7 +138,7 @@ func NotificationEventStreamHandler(conf config.SyncConfig, notificationReceiver continue } - jsonEvent, err := json.Marshal(event) + jsonEvent, err := json.Marshal(event.Message) if err != nil { middleware.GetLogger(r).Err(err).Msg("failed to marshal notification into json") continue @@ -159,108 +159,7 @@ func NotificationEventStreamHandler(conf config.SyncConfig, notificationReceiver } } -// NotificationEventSteamHandler implements the http.Handler interface. -func notificationEventSteamMonolithHandler(w http.ResponseWriter, r *http.Request) { - // Make sure that the writer supports flushing. - flusher, ok := w.(http.Flusher) - - if !ok { - http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) - return - } - - _, err := middleware.GetOptlyClient(r) - - if err != nil { - RenderError(err, http.StatusUnprocessableEntity, w, r) - return - } - - // Set the headers related to event streaming. - w.Header().Set("Content-Type", "text/event-stream") - w.Header().Set("Cache-Control", "no-cache") - w.Header().Set("Connection", "keep-alive") - - // Each connection registers its own message channel with the NotificationHandler's connections registry - messageChan := make(MessageChan) - // Each connection also adds listeners - sdkKey := r.Header.Get(middleware.OptlySDKHeader) - nc := registry.GetNotificationCenter(sdkKey) - - // Parse the form. - _ = r.ParseForm() - - filters := r.Form["filter"] - - // Parse out the any filters that were added - notificationsToAdd := getFilter(filters) - - ids := []struct { - int - notification.Type - }{} - - for notificationType, _ := range notificationsToAdd { - id, e := nc.AddHandler(notificationType, func(n interface{}) { - jsonEvent, err := json.Marshal(n) - if err != nil { - middleware.GetLogger(r).Error().Msg("encoding notification to json") - } else { - messageChan <- jsonEvent - } - }) - if e != nil { - RenderError(e, http.StatusUnprocessableEntity, w, r) - return - } - - // do defer outside the loop. - ids = append(ids, struct { - int - notification.Type - }{id, notificationType}) - } - - // Remove the decision listener if we exited. - defer func() { - for _, id := range ids { - err := nc.RemoveHandler(id.int, id.Type) - if err != nil { - middleware.GetLogger(r).Error().AnErr("removing notification", err) - } - } - }() - - // "raw" query string option - // If provided, send raw JSON lines instead of SSE-compliant strings. - raw := len(r.Form["raw"]) > 0 - - // Listen to connection close and un-register messageChan - notify := r.Context().Done() - // block waiting or messages broadcast on this connection's messageChan - for { - select { - // Write to the ResponseWriter - case msg := <-messageChan: - if raw { - // Raw JSON events, one per line - _, _ = fmt.Fprintf(w, "%s\n", msg) - } else { - // Server Sent Events compatible - _, _ = fmt.Fprintf(w, "data: %s\n\n", msg) - } - // Flush the data immediately instead of buffering it for later. - // The flush will fail if the connection is closed. That will cause the handler to exit. - flusher.Flush() - case <-notify: - middleware.GetLogger(r).Debug().Msg("received close on the request. So, we are shutting down this handler") - return - } - } - -} - -func DefaultNotificationReceiver(ctx context.Context, conf config.SyncConfig) (<-chan syncer.Notification, error) { +func DefaultNotificationReceiver(ctx context.Context) (<-chan syncer.Notification, error) { logger, ok := ctx.Value(LoggerKey).(*zerolog.Logger) if !ok { logger = &zerolog.Logger{} @@ -320,122 +219,54 @@ func DefaultNotificationReceiver(ctx context.Context, conf config.SyncConfig) (< return messageChan, nil } -func RedisNotificationReceiver(ctx context.Context, conf config.SyncConfig) (<-chan syncer.Notification, error) { - redisSyncer, err := syncer.NewRedisNotificationSyncer(&zerolog.Logger{}, conf) - if err != nil { - return nil, err - } - - client := redis.NewClient(&redis.Options{ - Addr: redisSyncer.Host, - Password: redisSyncer.Password, - DB: redisSyncer.Database, - }) - - // Subscribe to a Redis channel - pubsub := client.Subscribe(ctx, redisSyncer.Channel) - - dataChan := make(chan syncer.Notification) - - logger, ok := ctx.Value(LoggerKey).(*zerolog.Logger) - if !ok { - logger = &zerolog.Logger{} - } - - go func() { - for { - select { - case <-ctx.Done(): - client.Close() - pubsub.Close() - logger.Debug().Msg("context cancelled, redis notification receiver is closed") - return - default: - msg, err := pubsub.ReceiveMessage(ctx) - if err != nil { - logger.Err(err).Msg("failed to receive message from redis") - continue - } - - var event syncer.Notification - if err := json.Unmarshal([]byte(msg.Payload), &event); err != nil { - logger.Err(err).Msg("failed to unmarshal redis message") - continue - } - dataChan <- event - } - } - }() - - return dataChan, nil -} - -// NotificationEventSteamHandler implements the http.Handler interface. -func notificationEventSteamSyncHandler(conf config.SyncConfig, receiverFn NotificationReceiverFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - // Make sure that the writer supports flushing. - flusher, ok := w.(http.Flusher) - - if !ok { - http.Error(w, "Streaming unsupported!", http.StatusInternalServerError) - return +func RedisNotificationReceiver(conf config.SyncConfig) NotificationReceiverFunc { + return func(ctx context.Context) (<-chan syncer.Notification, error) { + redisSyncer, err := syncer.NewRedisNotificationSyncer(&zerolog.Logger{}, conf) + if err != nil { + return nil, err } - // Set the headers related to event streaming. - w.Header().Set("Content-Type", "text/event-stream") - w.Header().Set("Cache-Control", "no-cache") - w.Header().Set("Connection", "keep-alive") - - // "raw" query string option - // If provided, send raw JSON lines instead of SSE-compliant strings. - raw := len(r.Form["raw"]) > 0 - - // Parse the form. - _ = r.ParseForm() - - filters := r.Form["filter"] + client := redis.NewClient(&redis.Options{ + Addr: redisSyncer.Host, + Password: redisSyncer.Password, + DB: redisSyncer.Database, + }) - // Parse out the any filters that were added - notificationsToAdd := getFilter(filters) + // Subscribe to a Redis channel + pubsub := client.Subscribe(ctx, redisSyncer.Channel) - // Listen to connection close and un-register messageChan - notify := r.Context().Done() + dataChan := make(chan syncer.Notification) - dataChan, err := receiverFn(context.WithValue(r.Context(), LoggerKey, middleware.GetLogger(r)), conf) - if err != nil { - middleware.GetLogger(r).Err(err).Msg("error from receiver") - http.Error(w, "Error from data receiver!", http.StatusInternalServerError) - return + logger, ok := ctx.Value(LoggerKey).(*zerolog.Logger) + if !ok { + logger = &zerolog.Logger{} } - for { - select { - case <-notify: - middleware.GetLogger(r).Debug().Msg("received close on the request. So, we are shutting down this handler") - return - case event := <-dataChan: - _, found := notificationsToAdd[event.Type] - if !found { - continue - } - - jsonEvent, err := json.Marshal(event) - if err != nil { - middleware.GetLogger(r).Err(err).Msg("failed to marshal notification into json") - continue - } + go func() { + for { + select { + case <-ctx.Done(): + client.Close() + pubsub.Close() + logger.Debug().Msg("context cancelled, redis notification receiver is closed") + return + default: + msg, err := pubsub.ReceiveMessage(ctx) + if err != nil { + logger.Err(err).Msg("failed to receive message from redis") + continue + } - if raw { - // Raw JSON events, one per line - _, _ = fmt.Fprintf(w, "%s\n", string(jsonEvent)) - } else { - // Server Sent Events compatible - _, _ = fmt.Fprintf(w, "data: %s\n\n", string(jsonEvent)) + var event syncer.Notification + if err := json.Unmarshal([]byte(msg.Payload), &event); err != nil { + logger.Err(err).Msg("failed to unmarshal redis message") + continue + } + dataChan <- event } - // Flush the data immediately instead of buffering it for later. - // The flush will fail if the connection is closed. That will cause the handler to exit. - flusher.Flush() } - } + }() + + return dataChan, nil } } diff --git a/pkg/handlers/notification_test.go b/pkg/handlers/notification_test.go index baa71eaa..122ac1ea 100644 --- a/pkg/handlers/notification_test.go +++ b/pkg/handlers/notification_test.go @@ -31,6 +31,7 @@ import ( "github.com/optimizely/agent/pkg/middleware" "github.com/optimizely/agent/pkg/optimizely" "github.com/optimizely/agent/pkg/optimizely/optimizelytest" + "github.com/optimizely/agent/pkg/syncer" "github.com/go-chi/chi/v5" "github.com/optimizely/go-sdk/pkg/entities" @@ -74,7 +75,7 @@ func (suite *NotificationTestSuite) SetupTest() { func (suite *NotificationTestSuite) TestFeatureTestFilter() { conf := config.NewDefaultConfig() - suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(conf.Synchronization)) + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization))) feature := entities.Feature{Key: "one"} suite.tc.AddFeatureTest(feature) @@ -126,9 +127,6 @@ func (suite *NotificationTestSuite) TestFilter() { } func (suite *NotificationTestSuite) TestTrackAndProjectConfig() { - conf := config.NewDefaultConfig() - suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(conf.Synchronization)) - event := entities.Event{Key: "one"} suite.tc.AddEvent(event) @@ -143,16 +141,28 @@ func (suite *NotificationTestSuite) TestTrackAndProjectConfig() { nc := registry.GetNotificationCenter("") + notifications := make([]syncer.Notification, 0) + + trackEvent := map[string]string{"test": "value"} + projectConfigUpdateNotification := notification.ProjectConfigUpdateNotification{ + Type: notification.ProjectConfigUpdate, + Revision: suite.tc.ProjectConfig.GetRevision(), + } + + notifications = append(notifications, syncer.Notification{Type: notification.Track, Message: trackEvent}) + notifications = append(notifications, syncer.Notification{Type: notification.ProjectConfigUpdate, Message: projectConfigUpdateNotification}) + go func() { time.Sleep(1 * time.Second) - _ = nc.Send(notification.Track, map[string]string{"test": "value"}) - projectConfigUpdateNotification := notification.ProjectConfigUpdateNotification{ - Type: notification.ProjectConfigUpdate, - Revision: suite.tc.ProjectConfig.GetRevision(), - } + + _ = nc.Send(notification.Track, trackEvent) + _ = nc.Send(notification.ProjectConfigUpdate, projectConfigUpdateNotification) }() + conf := config.NewDefaultConfig() + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, notifications...))) + suite.mux.ServeHTTP(rec, req.WithContext(ctx1)) suite.Equal(http.StatusOK, rec.Code) @@ -163,22 +173,6 @@ func (suite *NotificationTestSuite) TestTrackAndProjectConfig() { } func (suite *NotificationTestSuite) TestTrackAndProjectConfigWithSynchronization() { - conf := config.NewDefaultConfig() - conf.Synchronization = config.SyncConfig{ - Notification: config.NotificationConfig{ - Enable: true, - Default: "redis", - Pubsub: map[string]interface{}{ - "redis": map[string]interface{}{ - "host": "localhost:6379", - "password": "", - "database": 0, - }, - }, - }, - } - suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(conf.Synchronization)) - event := entities.Event{Key: "one"} suite.tc.AddEvent(event) @@ -193,16 +187,41 @@ func (suite *NotificationTestSuite) TestTrackAndProjectConfigWithSynchronization nc := registry.GetNotificationCenter("") + notifications := make([]syncer.Notification, 0) + + trackEvent := map[string]string{"test": "value"} + projectConfigUpdateNotification := notification.ProjectConfigUpdateNotification{ + Type: notification.ProjectConfigUpdate, + Revision: suite.tc.ProjectConfig.GetRevision(), + } + + notifications = append(notifications, syncer.Notification{Type: notification.Track, Message: trackEvent}) + notifications = append(notifications, syncer.Notification{Type: notification.ProjectConfigUpdate, Message: projectConfigUpdateNotification}) + go func() { time.Sleep(1 * time.Second) - _ = nc.Send(notification.Track, map[string]string{"test": "value"}) - projectConfigUpdateNotification := notification.ProjectConfigUpdateNotification{ - Type: notification.ProjectConfigUpdate, - Revision: suite.tc.ProjectConfig.GetRevision(), - } + + _ = nc.Send(notification.Track, trackEvent) + _ = nc.Send(notification.ProjectConfigUpdate, projectConfigUpdateNotification) }() + conf := config.NewDefaultConfig() + conf.Synchronization = config.SyncConfig{ + Notification: config.NotificationConfig{ + Enable: true, + Default: "redis", + Pubsub: map[string]interface{}{ + "redis": map[string]interface{}{ + "host": "localhost:6379", + "password": "", + "database": 0, + }, + }, + }, + } + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, notifications...))) + suite.mux.ServeHTTP(rec, req.WithContext(ctx1)) suite.Equal(http.StatusOK, rec.Code) @@ -213,9 +232,6 @@ func (suite *NotificationTestSuite) TestTrackAndProjectConfigWithSynchronization } func (suite *NotificationTestSuite) TestActivateExperimentRaw() { - conf := config.NewDefaultConfig() - suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(conf.Synchronization)) - testVariation := suite.tc.ProjectConfig.CreateVariation("variation_a") suite.tc.AddExperiment("one", []entities.Variation{testVariation}) @@ -229,11 +245,19 @@ func (suite *NotificationTestSuite) TestActivateExperimentRaw() { ctx1, _ := context.WithTimeout(ctx, 2*time.Second) nc := registry.GetNotificationCenter("") + decisionEvent := map[string]string{"key": "value"} + + notifications := make([]syncer.Notification, 0) + notifications = append(notifications, syncer.Notification{Type: notification.Decision, Message: decisionEvent}) + go func() { time.Sleep(1 * time.Second) - nc.Send(notification.Decision, map[string]string{"key": "value"}) + nc.Send(notification.Decision, decisionEvent) }() + conf := config.NewDefaultConfig() + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, notifications...))) + suite.mux.ServeHTTP(rec, req.WithContext(ctx1)) suite.Equal(http.StatusOK, rec.Code) @@ -262,7 +286,7 @@ func TestEventStreamMissingOptlyCtx(t *testing.T) { conf := config.NewDefaultConfig() handlers := []func(w http.ResponseWriter, r *http.Request){ - NotificationEventStreamHandler(conf.Synchronization), + NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization)), } for _, handler := range handlers { @@ -271,3 +295,16 @@ func TestEventStreamMissingOptlyCtx(t *testing.T) { assertError(t, rec, "optlyClient not available", http.StatusUnprocessableEntity) } } + +func getMockNotificationReceiver(conf config.SyncConfig, msg ...syncer.Notification) NotificationReceiverFunc { + return func(ctx context.Context) (<-chan syncer.Notification, error) { + dataChan := make(chan syncer.Notification) + go func() { + time.Sleep(1) + for _, val := range msg { + dataChan <- val + } + }() + return dataChan, nil + } +} diff --git a/pkg/routers/api.go b/pkg/routers/api.go index 0c8886ef..29e152a7 100644 --- a/pkg/routers/api.go +++ b/pkg/routers/api.go @@ -84,9 +84,9 @@ func NewDefaultAPIRouter(optlyCache optimizely.Cache, conf *config.AgentConfig, nStreamHandler := forbiddenHandler("Notification stream not enabled") if conf.API.EnableNotifications { - nStreamHandler = handlers.NotificationEventStreamHandler(conf.Synchronization, handlers.DefaultNotificationReceiver) + nStreamHandler = handlers.NotificationEventStreamHandler(handlers.DefaultNotificationReceiver) if conf.Synchronization.Notification.Enable { - nStreamHandler = handlers.NotificationEventStreamHandler(conf.Synchronization, handlers.RedisNotificationReceiver) + nStreamHandler = handlers.NotificationEventStreamHandler(handlers.RedisNotificationReceiver(conf.Synchronization)) } } From 1055851d73bc3b079415e4702f5150ec13ad1a98 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Thu, 14 Sep 2023 22:05:38 +0600 Subject: [PATCH 22/33] add unit tests for default receiver --- pkg/handlers/notification.go | 7 ----- pkg/handlers/notification_test.go | 46 +++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 899b0f77..d0a1a3ae 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -73,13 +73,6 @@ func getFilter(filters []string) map[notification.Type]string { return notificationsToAdd } -// func NotificationEventStreamHandler(syncConfig config.SyncConfig) http.HandlerFunc { -// if syncConfig.Notification.Enable { -// return notificationEventSteamSyncHandler(syncConfig, redisNotificationReceiver) -// } -// return notificationEventSteamMonolithHandler -// } - func NotificationEventStreamHandler(notificationReceiverFn NotificationReceiverFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // Make sure that the writer supports flushing. diff --git a/pkg/handlers/notification_test.go b/pkg/handlers/notification_test.go index 122ac1ea..d729ec70 100644 --- a/pkg/handlers/notification_test.go +++ b/pkg/handlers/notification_test.go @@ -21,20 +21,19 @@ import ( "context" "net/http" "net/http/httptest" + "reflect" "testing" "time" - "github.com/optimizely/go-sdk/pkg/notification" - "github.com/optimizely/go-sdk/pkg/registry" - + "github.com/go-chi/chi/v5" "github.com/optimizely/agent/config" "github.com/optimizely/agent/pkg/middleware" "github.com/optimizely/agent/pkg/optimizely" "github.com/optimizely/agent/pkg/optimizely/optimizelytest" "github.com/optimizely/agent/pkg/syncer" - - "github.com/go-chi/chi/v5" "github.com/optimizely/go-sdk/pkg/entities" + "github.com/optimizely/go-sdk/pkg/notification" + "github.com/optimizely/go-sdk/pkg/registry" "github.com/stretchr/testify/suite" ) @@ -308,3 +307,40 @@ func getMockNotificationReceiver(conf config.SyncConfig, msg ...syncer.Notificat return dataChan, nil } } + +func TestDefaultNotificationReceiver(t *testing.T) { + type args struct { + ctx context.Context + } + tests := []struct { + name string + args args + want <-chan syncer.Notification + wantErr bool + }{ + { + name: "Test happy path", + args: args{ctx: context.WithValue(context.TODO(), SDKKey, "1221")}, + want: make(chan syncer.Notification), + wantErr: false, + }, + { + name: "Test without sdk key", + args: args{ctx: context.TODO()}, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := DefaultNotificationReceiver(tt.args.ctx) + if (err != nil) != tt.wantErr { + t.Errorf("DefaultNotificationReceiver() error = %v, wantErr %v", err, tt.wantErr) + return + } + if reflect.TypeOf(tt.want) != reflect.TypeOf(got) { + t.Errorf("DefaultNotificationReceiver() = %v, want %v", got, tt.want) + } + }) + } +} From 2d6e4280ebcb828b9c7379531d06e868d6e7cfd4 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Thu, 14 Sep 2023 22:31:23 +0600 Subject: [PATCH 23/33] add unit tests for redis receiver --- pkg/handlers/notification_test.go | 59 +++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/pkg/handlers/notification_test.go b/pkg/handlers/notification_test.go index d729ec70..89f46299 100644 --- a/pkg/handlers/notification_test.go +++ b/pkg/handlers/notification_test.go @@ -19,6 +19,7 @@ package handlers import ( "context" + "errors" "net/http" "net/http/httptest" "reflect" @@ -344,3 +345,61 @@ func TestDefaultNotificationReceiver(t *testing.T) { }) } } + +func TestRedisNotificationReceiver(t *testing.T) { + conf := config.SyncConfig{ + Notification: config.NotificationConfig{ + Enable: true, + Default: "redis", + Pubsub: map[string]interface{}{ + "redis": map[string]interface{}{ + "host": "localhost:6379", + "password": "", + "database": 0, + }, + }, + }, + } + type args struct { + conf config.SyncConfig + } + tests := []struct { + name string + args args + want NotificationReceiverFunc + }{ + { + name: "Test happy path", + args: args{conf: conf}, + want: func(ctx context.Context) (<-chan syncer.Notification, error) { + return make(<-chan syncer.Notification), nil + }, + }, + { + name: "Test empty config", + args: args{conf: config.SyncConfig{}}, + want: func(ctx context.Context) (<-chan syncer.Notification, error) { + return nil, errors.New("error") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := RedisNotificationReceiver(tt.args.conf) + if reflect.TypeOf(got) != reflect.TypeOf(tt.want) { + t.Errorf("RedisNotificationReceiver() = %v, want %v", got, tt.want) + } + + ch1, err1 := got(context.TODO()) + ch2, err2 := tt.want(context.TODO()) + + if reflect.TypeOf(err1) != reflect.TypeOf(err2) { + t.Errorf("error type not matched") + } + + if reflect.TypeOf(ch1) != reflect.TypeOf(ch2) { + t.Errorf("error type not matched") + } + }) + } +} From 1a18bb1b48b2f132712c11e9a356ba21e81a13ac Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Fri, 15 Sep 2023 18:17:25 +0600 Subject: [PATCH 24/33] add unit test --- pkg/handlers/notification_test.go | 66 ++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/pkg/handlers/notification_test.go b/pkg/handlers/notification_test.go index 89f46299..e408560e 100644 --- a/pkg/handlers/notification_test.go +++ b/pkg/handlers/notification_test.go @@ -75,7 +75,7 @@ func (suite *NotificationTestSuite) SetupTest() { func (suite *NotificationTestSuite) TestFeatureTestFilter() { conf := config.NewDefaultConfig() - suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization))) + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, false))) feature := entities.Feature{Key: "one"} suite.tc.AddFeatureTest(feature) @@ -161,7 +161,7 @@ func (suite *NotificationTestSuite) TestTrackAndProjectConfig() { }() conf := config.NewDefaultConfig() - suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, notifications...))) + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, false, notifications...))) suite.mux.ServeHTTP(rec, req.WithContext(ctx1)) @@ -220,7 +220,7 @@ func (suite *NotificationTestSuite) TestTrackAndProjectConfigWithSynchronization }, }, } - suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, notifications...))) + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, false, notifications...))) suite.mux.ServeHTTP(rec, req.WithContext(ctx1)) @@ -256,7 +256,7 @@ func (suite *NotificationTestSuite) TestActivateExperimentRaw() { }() conf := config.NewDefaultConfig() - suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, notifications...))) + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, false, notifications...))) suite.mux.ServeHTTP(rec, req.WithContext(ctx1)) @@ -267,6 +267,33 @@ func (suite *NotificationTestSuite) TestActivateExperimentRaw() { suite.Equal(expected, response) } +func (suite *NotificationTestSuite) TestWithFailedNotificationReceiver() { + req := httptest.NewRequest("GET", "/notifications/event-stream", nil) + rec := httptest.NewRecorder() + + // create a cancelable request context + ctx := req.Context() + ctx1, _ := context.WithTimeout(ctx, 2*time.Second) + + nc := registry.GetNotificationCenter("") + decisionEvent := map[string]string{"key": "value"} + + notifications := make([]syncer.Notification, 0) + notifications = append(notifications, syncer.Notification{Type: notification.Decision, Message: decisionEvent}) + + go func() { + time.Sleep(1 * time.Second) + nc.Send(notification.Decision, decisionEvent) + }() + + conf := config.NewDefaultConfig() + suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, true, notifications...))) + + suite.mux.ServeHTTP(rec, req.WithContext(ctx1)) + + suite.Equal(http.StatusInternalServerError, rec.Code) +} + func (suite *NotificationTestSuite) assertError(rec *httptest.ResponseRecorder, msg string, code int) { assertError(suite.T(), rec, msg, code) } @@ -286,7 +313,7 @@ func TestEventStreamMissingOptlyCtx(t *testing.T) { conf := config.NewDefaultConfig() handlers := []func(w http.ResponseWriter, r *http.Request){ - NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization)), + NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, false)), } for _, handler := range handlers { @@ -296,19 +323,6 @@ func TestEventStreamMissingOptlyCtx(t *testing.T) { } } -func getMockNotificationReceiver(conf config.SyncConfig, msg ...syncer.Notification) NotificationReceiverFunc { - return func(ctx context.Context) (<-chan syncer.Notification, error) { - dataChan := make(chan syncer.Notification) - go func() { - time.Sleep(1) - for _, val := range msg { - dataChan <- val - } - }() - return dataChan, nil - } -} - func TestDefaultNotificationReceiver(t *testing.T) { type args struct { ctx context.Context @@ -403,3 +417,19 @@ func TestRedisNotificationReceiver(t *testing.T) { }) } } + +func getMockNotificationReceiver(conf config.SyncConfig, returnError bool, msg ...syncer.Notification) NotificationReceiverFunc { + return func(ctx context.Context) (<-chan syncer.Notification, error) { + if returnError { + return nil, errors.New("mock error") + } + dataChan := make(chan syncer.Notification) + go func() { + time.Sleep(1) + for _, val := range msg { + dataChan <- val + } + }() + return dataChan, nil + } +} From 1a32830aa4fabc2e17598b4d138db9b34d4dfd60 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Fri, 15 Sep 2023 18:52:15 +0600 Subject: [PATCH 25/33] add comments --- config/config.go | 2 ++ pkg/handlers/notification.go | 16 +++++++------- pkg/handlers/notification_test.go | 36 +++++++++++++++---------------- pkg/syncer/syncer.go | 33 ++++++++++++++++++++++++---- 4 files changed, 57 insertions(+), 30 deletions(-) diff --git a/config/config.go b/config/config.go index a31ffdde..8bea3a6a 100644 --- a/config/config.go +++ b/config/config.go @@ -160,10 +160,12 @@ type AgentConfig struct { Synchronization SyncConfig `json:"synchronization"` } +// SyncConfig contains Synchronization configuration for the multiple Agent nodes type SyncConfig struct { Notification NotificationConfig `json:"notification"` } +// NotificationConfig contains Notification Synchronization configuration for the multiple Agent nodes type NotificationConfig struct { Enable bool `json:"enable"` Default string `json:"default"` diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index d0a1a3ae..495d94b4 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -43,7 +43,7 @@ const ( // Each http handler call creates a new channel and pumps decision service messages onto it. type MessageChan chan []byte -type NotificationReceiverFunc func(context.Context) (<-chan syncer.Notification, error) +type NotificationReceiverFunc func(context.Context) (<-chan syncer.Event, error) // types of notifications supported. var types = map[notification.Type]string{ @@ -152,7 +152,7 @@ func NotificationEventStreamHandler(notificationReceiverFn NotificationReceiverF } } -func DefaultNotificationReceiver(ctx context.Context) (<-chan syncer.Notification, error) { +func DefaultNotificationReceiver(ctx context.Context) (<-chan syncer.Event, error) { logger, ok := ctx.Value(LoggerKey).(*zerolog.Logger) if !ok { logger = &zerolog.Logger{} @@ -164,7 +164,7 @@ func DefaultNotificationReceiver(ctx context.Context) (<-chan syncer.Notificatio } // Each connection registers its own message channel with the NotificationHandler's connections registry - messageChan := make(chan syncer.Notification) + messageChan := make(chan syncer.Event) nc := registry.GetNotificationCenter(sdkKey) // Parse out the any filters that were added @@ -177,7 +177,7 @@ func DefaultNotificationReceiver(ctx context.Context) (<-chan syncer.Notificatio for notificationType := range notificationsToAdd { id, e := nc.AddHandler(notificationType, func(n interface{}) { - msg := syncer.Notification{ + msg := syncer.Event{ Type: notificationType, Message: n, } @@ -213,7 +213,7 @@ func DefaultNotificationReceiver(ctx context.Context) (<-chan syncer.Notificatio } func RedisNotificationReceiver(conf config.SyncConfig) NotificationReceiverFunc { - return func(ctx context.Context) (<-chan syncer.Notification, error) { + return func(ctx context.Context) (<-chan syncer.Event, error) { redisSyncer, err := syncer.NewRedisNotificationSyncer(&zerolog.Logger{}, conf) if err != nil { return nil, err @@ -228,7 +228,7 @@ func RedisNotificationReceiver(conf config.SyncConfig) NotificationReceiverFunc // Subscribe to a Redis channel pubsub := client.Subscribe(ctx, redisSyncer.Channel) - dataChan := make(chan syncer.Notification) + dataChan := make(chan syncer.Event) logger, ok := ctx.Value(LoggerKey).(*zerolog.Logger) if !ok { @@ -241,7 +241,7 @@ func RedisNotificationReceiver(conf config.SyncConfig) NotificationReceiverFunc case <-ctx.Done(): client.Close() pubsub.Close() - logger.Debug().Msg("context cancelled, redis notification receiver is closed") + logger.Debug().Msg("context canceled, redis notification receiver is closed") return default: msg, err := pubsub.ReceiveMessage(ctx) @@ -250,7 +250,7 @@ func RedisNotificationReceiver(conf config.SyncConfig) NotificationReceiverFunc continue } - var event syncer.Notification + var event syncer.Event if err := json.Unmarshal([]byte(msg.Payload), &event); err != nil { logger.Err(err).Msg("failed to unmarshal redis message") continue diff --git a/pkg/handlers/notification_test.go b/pkg/handlers/notification_test.go index e408560e..610b2774 100644 --- a/pkg/handlers/notification_test.go +++ b/pkg/handlers/notification_test.go @@ -141,7 +141,7 @@ func (suite *NotificationTestSuite) TestTrackAndProjectConfig() { nc := registry.GetNotificationCenter("") - notifications := make([]syncer.Notification, 0) + notifications := make([]syncer.Event, 0) trackEvent := map[string]string{"test": "value"} projectConfigUpdateNotification := notification.ProjectConfigUpdateNotification{ @@ -149,8 +149,8 @@ func (suite *NotificationTestSuite) TestTrackAndProjectConfig() { Revision: suite.tc.ProjectConfig.GetRevision(), } - notifications = append(notifications, syncer.Notification{Type: notification.Track, Message: trackEvent}) - notifications = append(notifications, syncer.Notification{Type: notification.ProjectConfigUpdate, Message: projectConfigUpdateNotification}) + notifications = append(notifications, syncer.Event{Type: notification.Track, Message: trackEvent}) + notifications = append(notifications, syncer.Event{Type: notification.ProjectConfigUpdate, Message: projectConfigUpdateNotification}) go func() { time.Sleep(1 * time.Second) @@ -187,7 +187,7 @@ func (suite *NotificationTestSuite) TestTrackAndProjectConfigWithSynchronization nc := registry.GetNotificationCenter("") - notifications := make([]syncer.Notification, 0) + notifications := make([]syncer.Event, 0) trackEvent := map[string]string{"test": "value"} projectConfigUpdateNotification := notification.ProjectConfigUpdateNotification{ @@ -195,8 +195,8 @@ func (suite *NotificationTestSuite) TestTrackAndProjectConfigWithSynchronization Revision: suite.tc.ProjectConfig.GetRevision(), } - notifications = append(notifications, syncer.Notification{Type: notification.Track, Message: trackEvent}) - notifications = append(notifications, syncer.Notification{Type: notification.ProjectConfigUpdate, Message: projectConfigUpdateNotification}) + notifications = append(notifications, syncer.Event{Type: notification.Track, Message: trackEvent}) + notifications = append(notifications, syncer.Event{Type: notification.ProjectConfigUpdate, Message: projectConfigUpdateNotification}) go func() { time.Sleep(1 * time.Second) @@ -247,8 +247,8 @@ func (suite *NotificationTestSuite) TestActivateExperimentRaw() { nc := registry.GetNotificationCenter("") decisionEvent := map[string]string{"key": "value"} - notifications := make([]syncer.Notification, 0) - notifications = append(notifications, syncer.Notification{Type: notification.Decision, Message: decisionEvent}) + notifications := make([]syncer.Event, 0) + notifications = append(notifications, syncer.Event{Type: notification.Decision, Message: decisionEvent}) go func() { time.Sleep(1 * time.Second) @@ -278,8 +278,8 @@ func (suite *NotificationTestSuite) TestWithFailedNotificationReceiver() { nc := registry.GetNotificationCenter("") decisionEvent := map[string]string{"key": "value"} - notifications := make([]syncer.Notification, 0) - notifications = append(notifications, syncer.Notification{Type: notification.Decision, Message: decisionEvent}) + notifications := make([]syncer.Event, 0) + notifications = append(notifications, syncer.Event{Type: notification.Decision, Message: decisionEvent}) go func() { time.Sleep(1 * time.Second) @@ -330,13 +330,13 @@ func TestDefaultNotificationReceiver(t *testing.T) { tests := []struct { name string args args - want <-chan syncer.Notification + want <-chan syncer.Event wantErr bool }{ { name: "Test happy path", args: args{ctx: context.WithValue(context.TODO(), SDKKey, "1221")}, - want: make(chan syncer.Notification), + want: make(chan syncer.Event), wantErr: false, }, { @@ -385,14 +385,14 @@ func TestRedisNotificationReceiver(t *testing.T) { { name: "Test happy path", args: args{conf: conf}, - want: func(ctx context.Context) (<-chan syncer.Notification, error) { - return make(<-chan syncer.Notification), nil + want: func(ctx context.Context) (<-chan syncer.Event, error) { + return make(<-chan syncer.Event), nil }, }, { name: "Test empty config", args: args{conf: config.SyncConfig{}}, - want: func(ctx context.Context) (<-chan syncer.Notification, error) { + want: func(ctx context.Context) (<-chan syncer.Event, error) { return nil, errors.New("error") }, }, @@ -418,12 +418,12 @@ func TestRedisNotificationReceiver(t *testing.T) { } } -func getMockNotificationReceiver(conf config.SyncConfig, returnError bool, msg ...syncer.Notification) NotificationReceiverFunc { - return func(ctx context.Context) (<-chan syncer.Notification, error) { +func getMockNotificationReceiver(conf config.SyncConfig, returnError bool, msg ...syncer.Event) NotificationReceiverFunc { + return func(ctx context.Context) (<-chan syncer.Event, error) { if returnError { return nil, errors.New("mock error") } - dataChan := make(chan syncer.Notification) + dataChan := make(chan syncer.Event) go func() { time.Sleep(1) for _, val := range msg { diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index 807416f1..e7d6ae65 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -1,3 +1,20 @@ +/**************************************************************************** + * Copyright 2019-2020,2023, Optimizely, Inc. and contributors * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ***************************************************************************/ + +// Package syncer provides synchronization across Agent nodes package syncer import ( @@ -12,15 +29,19 @@ import ( ) const ( + // PubSubDefaultChan will be used as default pubsub channel name PubSubDefaultChan = "optimizely-notifications" - PubSubRedis = "redis" + // PubSubRedis is the name of pubsub type of Redis + PubSubRedis = "redis" ) -type Notification struct { +// Event holds the notification event with it's type +type Event struct { Type notification.Type `json:"type"` Message interface{} `json:"message"` } +// RedisNotificationSyncer defines Redis pubsub configuration type RedisNotificationSyncer struct { Host string Password string @@ -29,6 +50,7 @@ type RedisNotificationSyncer struct { logger *zerolog.Logger } +// NewRedisNotificationSyncer returns an instance of RedisNotificationSyncer func NewRedisNotificationSyncer(logger *zerolog.Logger, conf config.SyncConfig) (*RedisNotificationSyncer, error) { if !conf.Notification.Enable { return nil, errors.New("notification syncer is not enabled") @@ -75,21 +97,24 @@ func NewRedisNotificationSyncer(logger *zerolog.Logger, conf config.SyncConfig) }, nil } +// AddHandler is empty but needed to implement notification.Center interface func (r *RedisNotificationSyncer) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { return 0, nil } +// RemoveHandler is empty but needed to implement notification.Center interface func (r *RedisNotificationSyncer) RemoveHandler(_ int, t notification.Type) error { return nil } +// Send will send the notification to the specified channel in the Redis pubsub func (r *RedisNotificationSyncer) Send(t notification.Type, n interface{}) error { - notification := Notification{ + event := Event{ Type: t, Message: n, } - jsonEvent, err := json.Marshal(notification) + jsonEvent, err := json.Marshal(event) if err != nil { return err } From e6eafa72573fccbeba43306b02385fe836110736 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Fri, 15 Sep 2023 19:09:02 +0600 Subject: [PATCH 26/33] clean up --- Dockerfile | 14 -------------- config.yaml | 4 ++-- redis.yaml | 33 --------------------------------- 3 files changed, 2 insertions(+), 49 deletions(-) delete mode 100644 Dockerfile delete mode 100644 redis.yaml diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 8e47d626..00000000 --- a/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM golang:1.21.0 as builder -RUN addgroup -u 1000 agentgroup &&\ - useradd -u 1000 agentuser -g agentgroup -WORKDIR /go/src/github.com/optimizely/agent -COPY . . -RUN make setup build &&\ - make ci_build_static_binary - -FROM scratch -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=builder /go/src/github.com/optimizely/agent/bin/optimizely /optimizely -COPY --from=builder /etc/passwd /etc/passwd -USER agentuser -CMD ["/optimizely"] diff --git a/config.yaml b/config.yaml index 1d89fa41..90b9c79a 100644 --- a/config.yaml +++ b/config.yaml @@ -47,7 +47,7 @@ server: readTimeout: 5s ## the maximum duration before timing out writes of the response. ## Value can be set in seconds (e.g. "5s") or milliseconds (e.g. "5000ms") - writeTimeout: -1 + writeTimeout: 10s ## path for the health status api healthCheckPath: "/health" ## the location of the TLS key file @@ -69,7 +69,7 @@ api: ## http listener port port: "8080" ## set to true to enable subscribing to notifications via an SSE event-stream - enableNotifications: true + enableNotifications: false ## set to true to be able to override experiment bucketing. (recommended false in production) enableOverrides: true ## CORS support is provided via chi middleware diff --git a/redis.yaml b/redis.yaml deleted file mode 100644 index b54c757a..00000000 --- a/redis.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: redis - namespace: demo -spec: - selector: - app: redis - ports: - - protocol: TCP - port: 6379 - targetPort: 6379 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: redis - namespace: demo -spec: - replicas: 1 - selector: - matchLabels: - app: redis - template: - metadata: - labels: - app: redis - spec: - containers: - - name: redis - image: redis:latest - ports: - - containerPort: 6379 From 6c34e842b8c89a0345c2fcae6271d0b7b6b89bac Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Fri, 15 Sep 2023 21:43:11 +0600 Subject: [PATCH 27/33] fix license header year --- pkg/handlers/notification.go | 2 +- pkg/syncer/syncer.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 495d94b4..3a6bf2b0 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -1,5 +1,5 @@ /**************************************************************************** - * Copyright 2020, Optimizely, Inc. and contributors * + * Copyright 2020,2023 Optimizely, Inc. and contributors * * * * Licensed under the Apache License, Version 2.0 (the "License"); * * you may not use this file except in compliance with the License. * diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index e7d6ae65..6c4318f1 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -1,5 +1,5 @@ /**************************************************************************** - * Copyright 2019-2020,2023, Optimizely, Inc. and contributors * + * Copyright 2023 Optimizely, Inc. and contributors * * * * Licensed under the Apache License, Version 2.0 (the "License"); * * you may not use this file except in compliance with the License. * From e364e21afcd8c512a81b989f5b67d2134290b70f Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Fri, 20 Oct 2023 17:08:20 +0600 Subject: [PATCH 28/33] update config --- config.yaml | 12 ++++++------ config/config.go | 32 ++++++++++++------------------- pkg/handlers/notification_test.go | 28 +++++++++++++-------------- pkg/syncer/syncer.go | 4 ++-- 4 files changed, 34 insertions(+), 42 deletions(-) diff --git a/config.yaml b/config.yaml index 20129ebf..4d45b5bf 100644 --- a/config.yaml +++ b/config.yaml @@ -249,12 +249,12 @@ runtime: ## if notification synchronization is enabled, then the active notification event-stream API ## will get the notifications from multiple replicas synchronization: + pubsub: + redis: + host: "redis.demo.svc:6379" + password: "" + database: 0 + channel: "optimizely-notifications" notification: enable: false default: "redis" - pubsub: - redis: - host: "redis.demo.svc:6379" - password: "" - database: 0 - channel: "optimizely-notifications" diff --git a/config/config.go b/config/config.go index e133a5dd..227ce202 100644 --- a/config/config.go +++ b/config/config.go @@ -127,17 +127,17 @@ func NewDefaultConfig() *AgentConfig { Port: "8085", }, Synchronization: SyncConfig{ + Pubsub: map[string]interface{}{ + "redis": map[string]interface{}{ + "host": "localhost:6379", + "password": "", + "database": 0, + "channel": "optimizely-notifications", + }, + }, Notification: NotificationConfig{ Enable: false, Default: "redis", - Pubsub: map[string]interface{}{ - "redis": map[string]interface{}{ - "host": "localhost:6379", - "password": "", - "database": 0, - "channel": "optimizely-notifications", - }, - }, }, }, } @@ -166,22 +166,14 @@ type AgentConfig struct { // SyncConfig contains Synchronization configuration for the multiple Agent nodes type SyncConfig struct { - Notification NotificationConfig `json:"notification"` + Pubsub map[string]interface{} `json:"pubsub"` + Notification NotificationConfig `json:"notification"` } // NotificationConfig contains Notification Synchronization configuration for the multiple Agent nodes type NotificationConfig struct { - Enable bool `json:"enable"` - Default string `json:"default"` - Pubsub map[string]interface{} `json:"pubsub"` - Admin AdminConfig `json:"admin"` - API APIConfig `json:"api"` - Log LogConfig `json:"log"` - Tracing TracingConfig `json:"tracing"` - Client ClientConfig `json:"client"` - Runtime RuntimeConfig `json:"runtime"` - Server ServerConfig `json:"server"` - Webhook WebhookConfig `json:"webhook"` + Enable bool `json:"enable"` + Default string `json:"default"` } // HTTPSDisabledWarning is logged when keyfile and certfile are not provided in server configuration diff --git a/pkg/handlers/notification_test.go b/pkg/handlers/notification_test.go index 610b2774..5c77e963 100644 --- a/pkg/handlers/notification_test.go +++ b/pkg/handlers/notification_test.go @@ -208,16 +208,16 @@ func (suite *NotificationTestSuite) TestTrackAndProjectConfigWithSynchronization conf := config.NewDefaultConfig() conf.Synchronization = config.SyncConfig{ + Pubsub: map[string]interface{}{ + "redis": map[string]interface{}{ + "host": "localhost:6379", + "password": "", + "database": 0, + }, + }, Notification: config.NotificationConfig{ Enable: true, Default: "redis", - Pubsub: map[string]interface{}{ - "redis": map[string]interface{}{ - "host": "localhost:6379", - "password": "", - "database": 0, - }, - }, }, } suite.mux.Get("/notifications/event-stream", NotificationEventStreamHandler(getMockNotificationReceiver(conf.Synchronization, false, notifications...))) @@ -362,16 +362,16 @@ func TestDefaultNotificationReceiver(t *testing.T) { func TestRedisNotificationReceiver(t *testing.T) { conf := config.SyncConfig{ + Pubsub: map[string]interface{}{ + "redis": map[string]interface{}{ + "host": "localhost:6379", + "password": "", + "database": 0, + }, + }, Notification: config.NotificationConfig{ Enable: true, Default: "redis", - Pubsub: map[string]interface{}{ - "redis": map[string]interface{}{ - "host": "localhost:6379", - "password": "", - "database": 0, - }, - }, }, } type args struct { diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index 6c4318f1..ac215ff3 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -58,11 +58,11 @@ func NewRedisNotificationSyncer(logger *zerolog.Logger, conf config.SyncConfig) if conf.Notification.Default != PubSubRedis { return nil, errors.New("redis syncer is not set as default") } - if conf.Notification.Pubsub == nil { + if conf.Pubsub == nil { return nil, errors.New("redis config is not given") } - redisConfig, found := conf.Notification.Pubsub[PubSubRedis].(map[string]interface{}) + redisConfig, found := conf.Pubsub[PubSubRedis].(map[string]interface{}) if !found { return nil, errors.New("redis pubsub config not found") } From b21f103b9bffbd9e10a31528fb34fa07106f3d02 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Mon, 23 Oct 2023 14:48:01 +0600 Subject: [PATCH 29/33] use default response writer --- pkg/middleware/trace.go | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/pkg/middleware/trace.go b/pkg/middleware/trace.go index 9b11768f..726e2007 100644 --- a/pkg/middleware/trace.go +++ b/pkg/middleware/trace.go @@ -80,16 +80,6 @@ func (gen *traceIDGenerator) NewIDs(ctx context.Context) (trace.TraceID, trace.S return tid, sid } -type statusRecorder struct { - http.ResponseWriter - statusCode int -} - -func (r *statusRecorder) WriteHeader(code int) { - r.statusCode = code - r.ResponseWriter.WriteHeader(code) -} - func AddTracing(conf config.TracingConfig, tracerName, spanName string) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { @@ -107,16 +97,7 @@ func AddTracing(conf config.TracingConfig, tracerName, spanName string) func(htt attribute.String(OptlySDKHeader, r.Header.Get(OptlySDKHeader)), ) - rec := &statusRecorder{ - ResponseWriter: w, - statusCode: http.StatusOK, - } - - next.ServeHTTP(rec, r.WithContext(ctx)) - - span.SetAttributes( - semconv.HTTPStatusCodeKey.Int(rec.statusCode), - ) + next.ServeHTTP(w, r.WithContext(ctx)) } return http.HandlerFunc(fn) } From 0e8a997eb6140b934d85ada95196572746eec173 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Mon, 23 Oct 2023 22:37:46 +0600 Subject: [PATCH 30/33] filter on sdk keys --- pkg/handlers/notification.go | 11 ++++++++--- pkg/optimizely/cache.go | 2 +- pkg/syncer/syncer.go | 38 ++++++++++++++++++++++++++++++++---- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 3a6bf2b0..47b82117 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -201,7 +201,7 @@ func DefaultNotificationReceiver(ctx context.Context) (<-chan syncer.Event, erro for _, id := range ids { err := nc.RemoveHandler(id.int, id.Type) if err != nil { - logger.Err(err).AnErr("removing notification", err) + logger.Err(err).AnErr("error in removing notification handler", err) } } return @@ -214,7 +214,12 @@ func DefaultNotificationReceiver(ctx context.Context) (<-chan syncer.Event, erro func RedisNotificationReceiver(conf config.SyncConfig) NotificationReceiverFunc { return func(ctx context.Context) (<-chan syncer.Event, error) { - redisSyncer, err := syncer.NewRedisNotificationSyncer(&zerolog.Logger{}, conf) + sdkKey, ok := ctx.Value(SDKKey).(string) + if !ok || sdkKey == "" { + return nil, errors.New("sdk key not found") + } + + redisSyncer, err := syncer.NewRedisNotificationSyncer(&zerolog.Logger{}, conf, sdkKey) if err != nil { return nil, err } @@ -226,7 +231,7 @@ func RedisNotificationReceiver(conf config.SyncConfig) NotificationReceiverFunc }) // Subscribe to a Redis channel - pubsub := client.Subscribe(ctx, redisSyncer.Channel) + pubsub := client.Subscribe(ctx, syncer.GetChannelForSDKKey(redisSyncer.Channel, sdkKey)) dataChan := make(chan syncer.Event) diff --git a/pkg/optimizely/cache.go b/pkg/optimizely/cache.go index 43ae35ad..1cb47012 100644 --- a/pkg/optimizely/cache.go +++ b/pkg/optimizely/cache.go @@ -252,7 +252,7 @@ func defaultLoader( } if agentConf.Synchronization.Notification.Enable { - redisSyncer, err := syncer.NewRedisNotificationSyncer(&zerolog.Logger{}, agentConf.Synchronization) + redisSyncer, err := syncer.NewRedisNotificationSyncer(&zerolog.Logger{}, agentConf.Synchronization, sdkKey) if err != nil { return nil, err } diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index ac215ff3..0c624b59 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -21,6 +21,8 @@ import ( "context" "encoding/json" "errors" + "fmt" + "sync" "github.com/go-redis/redis/v8" "github.com/optimizely/agent/config" @@ -35,6 +37,11 @@ const ( PubSubRedis = "redis" ) +var ( + ncCache = make(map[string]*RedisNotificationSyncer) + mutexLock = &sync.Mutex{} +) + // Event holds the notification event with it's type type Event struct { Type notification.Type `json:"type"` @@ -43,15 +50,24 @@ type Event struct { // RedisNotificationSyncer defines Redis pubsub configuration type RedisNotificationSyncer struct { + ctx context.Context Host string Password string Database int Channel string logger *zerolog.Logger + sdkKey string } // NewRedisNotificationSyncer returns an instance of RedisNotificationSyncer -func NewRedisNotificationSyncer(logger *zerolog.Logger, conf config.SyncConfig) (*RedisNotificationSyncer, error) { +func NewRedisNotificationSyncer(logger *zerolog.Logger, conf config.SyncConfig, sdkKey string) (*RedisNotificationSyncer, error) { + mutexLock.Lock() + defer mutexLock.Unlock() + + if nc, found := ncCache[sdkKey]; found { + return nc, nil + } + if !conf.Notification.Enable { return nil, errors.New("notification syncer is not enabled") } @@ -88,13 +104,22 @@ func NewRedisNotificationSyncer(logger *zerolog.Logger, conf config.SyncConfig) logger = &zerolog.Logger{} } - return &RedisNotificationSyncer{ + nc := &RedisNotificationSyncer{ + ctx: context.Background(), Host: host, Password: password, Database: database, Channel: channel, logger: logger, - }, nil + sdkKey: sdkKey, + } + ncCache[sdkKey] = nc + return nc, nil +} + +func (r *RedisNotificationSyncer) WithContext(ctx context.Context) *RedisNotificationSyncer { + r.ctx = ctx + return r } // AddHandler is empty but needed to implement notification.Center interface @@ -125,10 +150,15 @@ func (r *RedisNotificationSyncer) Send(t notification.Type, n interface{}) error DB: r.Database, }) defer client.Close() + channel := GetChannelForSDKKey(r.Channel, r.sdkKey) - if err := client.Publish(context.TODO(), r.Channel, jsonEvent).Err(); err != nil { + if err := client.Publish(r.ctx, channel, jsonEvent).Err(); err != nil { r.logger.Err(err).Msg("failed to publish json event to pub/sub") return err } return nil } + +func GetChannelForSDKKey(channel, key string) string { + return fmt.Sprintf("%s-%s", channel, key) +} From ce403f7bd67afff8c25ae12c9b57511fd7f6c1a1 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Mon, 23 Oct 2023 23:44:04 +0600 Subject: [PATCH 31/33] update unit test --- pkg/handlers/notification_test.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/handlers/notification_test.go b/pkg/handlers/notification_test.go index 5c77e963..82d0b95e 100644 --- a/pkg/handlers/notification_test.go +++ b/pkg/handlers/notification_test.go @@ -20,6 +20,7 @@ package handlers import ( "context" "errors" + "fmt" "net/http" "net/http/httptest" "reflect" @@ -376,6 +377,7 @@ func TestRedisNotificationReceiver(t *testing.T) { } type args struct { conf config.SyncConfig + ctx context.Context } tests := []struct { name string @@ -384,14 +386,20 @@ func TestRedisNotificationReceiver(t *testing.T) { }{ { name: "Test happy path", - args: args{conf: conf}, + args: args{ + conf: conf, + ctx: context.WithValue(context.Background(), SDKKey, "random-sdk-key-1"), + }, want: func(ctx context.Context) (<-chan syncer.Event, error) { return make(<-chan syncer.Event), nil }, }, { name: "Test empty config", - args: args{conf: config.SyncConfig{}}, + args: args{ + conf: config.SyncConfig{}, + ctx: context.WithValue(context.Background(), SDKKey, "random-sdk-key-2"), + }, want: func(ctx context.Context) (<-chan syncer.Event, error) { return nil, errors.New("error") }, @@ -404,15 +412,16 @@ func TestRedisNotificationReceiver(t *testing.T) { t.Errorf("RedisNotificationReceiver() = %v, want %v", got, tt.want) } - ch1, err1 := got(context.TODO()) - ch2, err2 := tt.want(context.TODO()) + ch1, err1 := got(tt.args.ctx) + ch2, err2 := tt.want(tt.args.ctx) if reflect.TypeOf(err1) != reflect.TypeOf(err2) { + fmt.Println(err1, err2) t.Errorf("error type not matched") } if reflect.TypeOf(ch1) != reflect.TypeOf(ch2) { - t.Errorf("error type not matched") + t.Errorf("channel type not matched") } }) } From 0ec42f2a49e25a6ab069d5f7b3e74bb8792aa285 Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Mon, 23 Oct 2023 23:56:18 +0600 Subject: [PATCH 32/33] fix failing acceptance test --- tests/acceptance/test_acceptance/test_odp_redis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/acceptance/test_acceptance/test_odp_redis.py b/tests/acceptance/test_acceptance/test_odp_redis.py index a58e2db2..7e4b5fdc 100644 --- a/tests/acceptance/test_acceptance/test_odp_redis.py +++ b/tests/acceptance/test_acceptance/test_odp_redis.py @@ -52,6 +52,7 @@ def test_redis_save(session_override_sdk_key_odp): """ expected_segments = ["atsbugbashsegmenthaspurchased", "atsbugbashsegmentdob"] + expected_segments_rev = ["atsbugbashsegmentdob", "atsbugbashsegmenthaspurchased"] uId = "fs_user_id-$-matjaz-user-1" r = redis.Redis(host='localhost', port=6379, db=0) # clean redis before testing since several tests use same user_id @@ -72,7 +73,7 @@ def test_redis_save(session_override_sdk_key_odp): params=params) # Check saved segments - assert json.loads(json.dumps(expected_segments)) == json.loads(r.get(uId)) + assert json.loads(json.dumps(expected_segments)) == json.loads(r.get(uId)) or json.loads(json.dumps(expected_segments_rev)) == json.loads(r.get(uId)) assert json.loads(json.dumps(expected_redis_save)) == resp.json() assert resp.status_code == 200, resp.text From d8744a265ba4a2c5282a70ca5a3ea00a4800105e Mon Sep 17 00:00:00 2001 From: Pulak Bhowmick Date: Mon, 6 Nov 2023 17:17:00 +0600 Subject: [PATCH 33/33] rename --- config.yaml | 2 +- pkg/handlers/notification.go | 2 +- pkg/optimizely/cache.go | 2 +- pkg/syncer/syncer.go | 22 +++++++++++----------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/config.yaml b/config.yaml index 66780ae3..87b872a0 100644 --- a/config.yaml +++ b/config.yaml @@ -253,7 +253,7 @@ synchronization: host: "redis.demo.svc:6379" password: "" database: 0 - channel: "optimizely-notifications" + channel: "optimizely-sync" notification: enable: false default: "redis" diff --git a/pkg/handlers/notification.go b/pkg/handlers/notification.go index 47b82117..5c391cf8 100644 --- a/pkg/handlers/notification.go +++ b/pkg/handlers/notification.go @@ -219,7 +219,7 @@ func RedisNotificationReceiver(conf config.SyncConfig) NotificationReceiverFunc return nil, errors.New("sdk key not found") } - redisSyncer, err := syncer.NewRedisNotificationSyncer(&zerolog.Logger{}, conf, sdkKey) + redisSyncer, err := syncer.NewRedisSyncer(&zerolog.Logger{}, conf, sdkKey) if err != nil { return nil, err } diff --git a/pkg/optimizely/cache.go b/pkg/optimizely/cache.go index 1cb47012..f1401311 100644 --- a/pkg/optimizely/cache.go +++ b/pkg/optimizely/cache.go @@ -252,7 +252,7 @@ func defaultLoader( } if agentConf.Synchronization.Notification.Enable { - redisSyncer, err := syncer.NewRedisNotificationSyncer(&zerolog.Logger{}, agentConf.Synchronization, sdkKey) + redisSyncer, err := syncer.NewRedisSyncer(&zerolog.Logger{}, agentConf.Synchronization, sdkKey) if err != nil { return nil, err } diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go index 0c624b59..aa102188 100644 --- a/pkg/syncer/syncer.go +++ b/pkg/syncer/syncer.go @@ -32,13 +32,13 @@ import ( const ( // PubSubDefaultChan will be used as default pubsub channel name - PubSubDefaultChan = "optimizely-notifications" + PubSubDefaultChan = "optimizely-sync" // PubSubRedis is the name of pubsub type of Redis PubSubRedis = "redis" ) var ( - ncCache = make(map[string]*RedisNotificationSyncer) + ncCache = make(map[string]*RedisSyncer) mutexLock = &sync.Mutex{} ) @@ -48,8 +48,8 @@ type Event struct { Message interface{} `json:"message"` } -// RedisNotificationSyncer defines Redis pubsub configuration -type RedisNotificationSyncer struct { +// RedisSyncer defines Redis pubsub configuration +type RedisSyncer struct { ctx context.Context Host string Password string @@ -59,8 +59,8 @@ type RedisNotificationSyncer struct { sdkKey string } -// NewRedisNotificationSyncer returns an instance of RedisNotificationSyncer -func NewRedisNotificationSyncer(logger *zerolog.Logger, conf config.SyncConfig, sdkKey string) (*RedisNotificationSyncer, error) { +// NewRedisSyncer returns an instance of RedisNotificationSyncer +func NewRedisSyncer(logger *zerolog.Logger, conf config.SyncConfig, sdkKey string) (*RedisSyncer, error) { mutexLock.Lock() defer mutexLock.Unlock() @@ -104,7 +104,7 @@ func NewRedisNotificationSyncer(logger *zerolog.Logger, conf config.SyncConfig, logger = &zerolog.Logger{} } - nc := &RedisNotificationSyncer{ + nc := &RedisSyncer{ ctx: context.Background(), Host: host, Password: password, @@ -117,23 +117,23 @@ func NewRedisNotificationSyncer(logger *zerolog.Logger, conf config.SyncConfig, return nc, nil } -func (r *RedisNotificationSyncer) WithContext(ctx context.Context) *RedisNotificationSyncer { +func (r *RedisSyncer) WithContext(ctx context.Context) *RedisSyncer { r.ctx = ctx return r } // AddHandler is empty but needed to implement notification.Center interface -func (r *RedisNotificationSyncer) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { +func (r *RedisSyncer) AddHandler(_ notification.Type, _ func(interface{})) (int, error) { return 0, nil } // RemoveHandler is empty but needed to implement notification.Center interface -func (r *RedisNotificationSyncer) RemoveHandler(_ int, t notification.Type) error { +func (r *RedisSyncer) RemoveHandler(_ int, t notification.Type) error { return nil } // Send will send the notification to the specified channel in the Redis pubsub -func (r *RedisNotificationSyncer) Send(t notification.Type, n interface{}) error { +func (r *RedisSyncer) Send(t notification.Type, n interface{}) error { event := Event{ Type: t, Message: n,