From 5cfb10e4476197b188ac83e504d2dd1d083769c7 Mon Sep 17 00:00:00 2001 From: Peter Sanchez Date: Thu, 19 Dec 2024 11:36:19 -0600 Subject: [PATCH] Update gobwebs and dowork. Changes to use the new version of dowork --- cmd/api/main.go | 9 +++++++-- cmd/links/main.go | 11 ++++++++--- cmd/list/main.go | 9 +++++++-- cmd/server.go | 21 +++++++++++++++++++++ cmd/short/main.go | 9 +++++++-- cmd/test/helpers.go | 4 ++-- config.example.ini | 6 ++++++ go.mod | 4 ++-- go.sum | 8 ++++---- 9 files changed, 64 insertions(+), 17 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 53b0365..4b3a7b7 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -82,6 +82,11 @@ func run() error { return fmt.Errorf("unable to load storage service: %v", err) } + eSize, gSize, err := cmd.LoadWorkerQueueSizes(config) + if err != nil { + return err + } + rlConfig, err := links.NewRateLimiterConfig(config) if err != nil { return err @@ -103,10 +108,10 @@ func run() error { e.IPExtractor = echo.ExtractIPFromXFFHeader() // email work queue and service, general task queue - eq := email.NewQueue() + eq := email.NewQueue(eSize) mailChecker := feedback.MailChecker{DB: db} sq := email.NewServiceQueue(e.Logger, esvc, eq, &mailChecker) - wq := work.NewQueue("general") + wq := work.NewQueue("general", gSize) mwConf := &server.MiddlewareConfig{ Sessions: false, diff --git a/cmd/links/main.go b/cmd/links/main.go index 658af8e..2a23479 100644 --- a/cmd/links/main.go +++ b/cmd/links/main.go @@ -160,6 +160,11 @@ func run() error { return fmt.Errorf("Unknown storage service configured") } + eSize, gSize, err := cmd.LoadWorkerQueueSizes(config) + if err != nil { + return err + } + tlsman := cmd.LoadAutoTLS(config, db, models.DomainServiceLinks) rlConfig, err := links.NewRateLimiterConfig(config) @@ -173,11 +178,11 @@ func run() error { e.IPExtractor = echo.ExtractIPFromXFFHeader() // email work queue and service, general task queue - eq := email.NewQueue() + eq := email.NewQueue(eSize) mailChecker := feedback.MailChecker{DB: db} sq := email.NewServiceQueue(e.Logger, esvc, eq, &mailChecker) - wq := work.NewQueue("general") - wqi := work.NewQueue("invoice") + wq := work.NewQueue("general", gSize) + wqi := work.NewQueue("invoice", gSize) mwConf := &server.MiddlewareConfig{ Sessions: true, diff --git a/cmd/list/main.go b/cmd/list/main.go index 1915cdd..7bd3d3b 100644 --- a/cmd/list/main.go +++ b/cmd/list/main.go @@ -59,6 +59,11 @@ func run() error { } } + eSize, gSize, err := cmd.LoadWorkerQueueSizes(config) + if err != nil { + return err + } + db, err := cmd.OpenDB(config) if err != nil { return fmt.Errorf("Unable to open connection to PostgreSQL: %v", err) @@ -70,8 +75,8 @@ func run() error { // Deployed via Caddy at the moment which uses X-Forwarded-For header by default e.IPExtractor = echo.ExtractIPFromXFFHeader() - eq := email.NewQueue() - wq := work.NewQueue("general") + eq := email.NewQueue(eSize) + wq := work.NewQueue("general", gSize) mwConf := &server.MiddlewareConfig{ Sessions: true, diff --git a/cmd/server.go b/cmd/server.go index eae660a..f64597d 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -206,3 +206,24 @@ func LoadAutoTLS(config *config.Config, db *sql.DB, service string) *autocert.Ma HostPolicy: domain.DomainHostPolicy(db, service), } } + +// LoadWorkerQueueSizes ... +func LoadWorkerQueueSizes(config *config.Config) (int, int, error) { + var ( + eSize, gSize int = 512, 512 + err error + ) + if tStr, ok := config.File.Get("links", "email-queue-size"); ok { + eSize, err = strconv.Atoi(tStr) + if err != nil { + return eSize, gSize, fmt.Errorf("links:email-queue-size invalid") + } + } + if tStr, ok := config.File.Get("links", "general-queue-size"); ok { + gSize, err = strconv.Atoi(tStr) + if err != nil { + return eSize, gSize, fmt.Errorf("links:general-queue-size invalid") + } + } + return eSize, gSize, nil +} diff --git a/cmd/short/main.go b/cmd/short/main.go index ff3aa19..3d5217e 100644 --- a/cmd/short/main.go +++ b/cmd/short/main.go @@ -57,6 +57,11 @@ func run() error { } } + eSize, gSize, err := cmd.LoadWorkerQueueSizes(config) + if err != nil { + return err + } + db, err := cmd.OpenDB(config) if err != nil { return fmt.Errorf("Unable to open connection to PostgreSQL: %v", err) @@ -68,8 +73,8 @@ func run() error { // Deployed via Caddy at the moment which uses X-Forwarded-For header by default e.IPExtractor = echo.ExtractIPFromXFFHeader() - eq := email.NewQueue() - wq := work.NewQueue("general") + eq := email.NewQueue(eSize) + wq := work.NewQueue("general", gSize) mwConf := &server.MiddlewareConfig{ Sessions: false, diff --git a/cmd/test/helpers.go b/cmd/test/helpers.go index f002b5e..382ecf6 100644 --- a/cmd/test/helpers.go +++ b/cmd/test/helpers.go @@ -62,9 +62,9 @@ func newTestServer(t *testing.T, name string, config *config.Config) (*server.Se e := echo.New() e.Logger.SetOutput(io.Discard) esvc := carrier.NewOutboxService() - eq := email.NewQueue() + eq := email.NewQueue(3) mailChecker := feedback.MailChecker{DB: db} - wq := work.NewQueue("general") + wq := work.NewQueue("general", 3) sq := email.NewServiceQueue(e.Logger, esvc, eq, &mailChecker) srv := server.New(e, db, config). Initialize(). diff --git a/config.example.ini b/config.example.ini index 8b591d3..caa93e7 100644 --- a/config.example.ini +++ b/config.example.ini @@ -182,6 +182,12 @@ rate-limit-burst=40 # How long (in minutes) of inactivity does the limit record live for rate-limit-expire=3 +# How many email queue workers. Defaults to 512 +email-queue-size = 512 + +# How many general queue workers. Defaults to 512 +general-queue-size = 512 + [stripe] secret-key= public-key= diff --git a/go.mod b/go.mod index e6b1975..098a5ac 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.5 require ( git.sr.ht/~emersion/gqlclient v0.0.0-20230820050442-8873fe0204b9 - git.sr.ht/~sircmpwn/dowork v0.0.0-20221010085743-46c4299d76a1 + git.sr.ht/~sircmpwn/dowork v0.0.0-20241219115748-a3928eb084db github.com/99designs/gqlgen v0.17.57 github.com/Masterminds/squirrel v1.5.4 github.com/alexedwards/scs/postgresstore v0.0.0-20240316134038-7e11d57e8885 @@ -28,7 +28,7 @@ require ( golang.org/x/text v0.20.0 golang.org/x/time v0.5.0 hg.code.netlandish.com/~netlandish/sendygo v0.0.0-20230124192435-bbf347776232 - netlandish.com/x/gobwebs v0.0.0-20240828125517-6fc18f8bc093 + netlandish.com/x/gobwebs v0.0.0-20241219173153-b82c01404952 netlandish.com/x/gobwebs-formguard v0.0.0-20240917220134-4b6f2ec7d1f5 netlandish.com/x/gobwebs-graphql v0.0.0-20241126234432-2cc59b2f7ebd netlandish.com/x/gobwebs-oauth2 v0.0.0-20240216010400-f0f42e2e3a7c diff --git a/go.sum b/go.sum index 6ca096d..f8c65f1 100644 --- a/go.sum +++ b/go.sum @@ -65,8 +65,8 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= git.sr.ht/~emersion/gqlclient v0.0.0-20230820050442-8873fe0204b9 h1:QNwHP6WknvS7X6MEFxCpefQb1QJMqgIIt+vn/PVoMMg= git.sr.ht/~emersion/gqlclient v0.0.0-20230820050442-8873fe0204b9/go.mod h1:kvl/JK0Z3VRmtbBxdOJR4ydyXVouUIcFIXgv4H6rVAY= -git.sr.ht/~sircmpwn/dowork v0.0.0-20221010085743-46c4299d76a1 h1:EvPKkneKkF/f7zEgKPqIZVyj3jWO8zSmsBOvMhAGqMA= -git.sr.ht/~sircmpwn/dowork v0.0.0-20221010085743-46c4299d76a1/go.mod h1:8neHEO3503w/rNtttnR0JFpQgM/GFhaafVwvkPsFIDw= +git.sr.ht/~sircmpwn/dowork v0.0.0-20241219115748-a3928eb084db h1:NX+qg2ZkefnAWL1TvdgumYBggN2f2zYGYQJxIaqaQ5s= +git.sr.ht/~sircmpwn/dowork v0.0.0-20241219115748-a3928eb084db/go.mod h1:8neHEO3503w/rNtttnR0JFpQgM/GFhaafVwvkPsFIDw= git.sr.ht/~sircmpwn/getopt v0.0.0-20191230200459-23622cc906b3/go.mod h1:wMEGFFFNuPos7vHmWXfszqImLppbc0wEhh6JBfJIUgw= git.sr.ht/~sircmpwn/go-bare v0.0.0-20210406120253-ab86bc2846d9 h1:Ahny8Ud1LjVMMAlt8utUFKhhxJtwBAualvsbc/Sk7cE= git.sr.ht/~sircmpwn/go-bare v0.0.0-20210406120253-ab86bc2846d9/go.mod h1:BVJwbDfVjCjoFiKrhkei6NdGcZYpkDkdyCdg1ukytRA= @@ -2584,8 +2584,8 @@ modernc.org/z v1.0.1-0.20210308123920-1f282aa71362/go.mod h1:8/SRk5C/HgiQWCgXdfp modernc.org/z v1.0.1/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA= modernc.org/z v1.2.20/go.mod h1:zU9FiF4PbHdOTUxw+IF8j7ArBMRPsHgq10uVPt6xTzo= modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4= -netlandish.com/x/gobwebs v0.0.0-20240828125517-6fc18f8bc093 h1:iUxk5pguGAV6d9C5HXj0/MoEI4gfrrwmqkaRMDIgX9Y= -netlandish.com/x/gobwebs v0.0.0-20240828125517-6fc18f8bc093/go.mod h1:9uw5D9T02tPpMMr58nThmVp8ZcdrRIAq/SAHsOIim5g= +netlandish.com/x/gobwebs v0.0.0-20241219173153-b82c01404952 h1:DtzTNLe5MtlYVZuOT0zANWFddsRzM1np4FuMkA/Y0tU= +netlandish.com/x/gobwebs v0.0.0-20241219173153-b82c01404952/go.mod h1:Q/e3xSbcMqYMpYS360pz5gAA0/cCRrBxNHnCAYODHoA= netlandish.com/x/gobwebs-formguard v0.0.0-20240917220134-4b6f2ec7d1f5 h1:z5hzVB0R+k2+bh8yuM7XeojGesI2i5tnQHrBNzxG2Xw= netlandish.com/x/gobwebs-formguard v0.0.0-20240917220134-4b6f2ec7d1f5/go.mod h1:mmpJMveuFlwAj3URXlNmVox3WYCn4OVlozFQdd95wH8= netlandish.com/x/gobwebs-graphql v0.0.0-20241126234432-2cc59b2f7ebd h1:QJzQj/+2XAEHM0mCxqYleV0sVM2tNZBS6zP6uuOB+Tk= -- 2.45.2