M cmd/api/main.go => cmd/api/main.go +7 -2
@@ 82,6 82,11 @@ func run() error {
return fmt.Errorf("unable to load storage service: %v", err)
}
+ eSize, gSize, err := cmd.LoadWorkerQueueSizes(config)
+ if err != nil {
+ return err
+ }
+
rlConfig, err := links.NewRateLimiterConfig(config)
if err != nil {
return err
@@ 103,10 108,10 @@ func run() error {
e.IPExtractor = echo.ExtractIPFromXFFHeader()
// email work queue and service, general task queue
- eq := email.NewQueue()
+ eq := email.NewQueue(eSize)
mailChecker := feedback.MailChecker{DB: db}
sq := email.NewServiceQueue(e.Logger, esvc, eq, &mailChecker)
- wq := work.NewQueue("general")
+ wq := work.NewQueue("general", gSize)
mwConf := &server.MiddlewareConfig{
Sessions: false,
M cmd/links/main.go => cmd/links/main.go +8 -3
@@ 160,6 160,11 @@ func run() error {
return fmt.Errorf("Unknown storage service configured")
}
+ eSize, gSize, err := cmd.LoadWorkerQueueSizes(config)
+ if err != nil {
+ return err
+ }
+
tlsman := cmd.LoadAutoTLS(config, db, models.DomainServiceLinks)
rlConfig, err := links.NewRateLimiterConfig(config)
@@ 173,11 178,11 @@ func run() error {
e.IPExtractor = echo.ExtractIPFromXFFHeader()
// email work queue and service, general task queue
- eq := email.NewQueue()
+ eq := email.NewQueue(eSize)
mailChecker := feedback.MailChecker{DB: db}
sq := email.NewServiceQueue(e.Logger, esvc, eq, &mailChecker)
- wq := work.NewQueue("general")
- wqi := work.NewQueue("invoice")
+ wq := work.NewQueue("general", gSize)
+ wqi := work.NewQueue("invoice", gSize)
mwConf := &server.MiddlewareConfig{
Sessions: true,
M cmd/list/main.go => cmd/list/main.go +7 -2
@@ 59,6 59,11 @@ func run() error {
}
}
+ eSize, gSize, err := cmd.LoadWorkerQueueSizes(config)
+ if err != nil {
+ return err
+ }
+
db, err := cmd.OpenDB(config)
if err != nil {
return fmt.Errorf("Unable to open connection to PostgreSQL: %v", err)
@@ 70,8 75,8 @@ func run() error {
// Deployed via Caddy at the moment which uses X-Forwarded-For header by default
e.IPExtractor = echo.ExtractIPFromXFFHeader()
- eq := email.NewQueue()
- wq := work.NewQueue("general")
+ eq := email.NewQueue(eSize)
+ wq := work.NewQueue("general", gSize)
mwConf := &server.MiddlewareConfig{
Sessions: true,
M cmd/server.go => cmd/server.go +21 -0
@@ 206,3 206,24 @@ func LoadAutoTLS(config *config.Config, db *sql.DB, service string) *autocert.Ma
HostPolicy: domain.DomainHostPolicy(db, service),
}
}
+
+// LoadWorkerQueueSizes ...
+func LoadWorkerQueueSizes(config *config.Config) (int, int, error) {
+ var (
+ eSize, gSize int = 512, 512
+ err error
+ )
+ if tStr, ok := config.File.Get("links", "email-queue-size"); ok {
+ eSize, err = strconv.Atoi(tStr)
+ if err != nil {
+ return eSize, gSize, fmt.Errorf("links:email-queue-size invalid")
+ }
+ }
+ if tStr, ok := config.File.Get("links", "general-queue-size"); ok {
+ gSize, err = strconv.Atoi(tStr)
+ if err != nil {
+ return eSize, gSize, fmt.Errorf("links:general-queue-size invalid")
+ }
+ }
+ return eSize, gSize, nil
+}
M cmd/short/main.go => cmd/short/main.go +7 -2
@@ 57,6 57,11 @@ func run() error {
}
}
+ eSize, gSize, err := cmd.LoadWorkerQueueSizes(config)
+ if err != nil {
+ return err
+ }
+
db, err := cmd.OpenDB(config)
if err != nil {
return fmt.Errorf("Unable to open connection to PostgreSQL: %v", err)
@@ 68,8 73,8 @@ func run() error {
// Deployed via Caddy at the moment which uses X-Forwarded-For header by default
e.IPExtractor = echo.ExtractIPFromXFFHeader()
- eq := email.NewQueue()
- wq := work.NewQueue("general")
+ eq := email.NewQueue(eSize)
+ wq := work.NewQueue("general", gSize)
mwConf := &server.MiddlewareConfig{
Sessions: false,
M cmd/test/helpers.go => cmd/test/helpers.go +2 -2
@@ 62,9 62,9 @@ func newTestServer(t *testing.T, name string, config *config.Config) (*server.Se
e := echo.New()
e.Logger.SetOutput(io.Discard)
esvc := carrier.NewOutboxService()
- eq := email.NewQueue()
+ eq := email.NewQueue(3)
mailChecker := feedback.MailChecker{DB: db}
- wq := work.NewQueue("general")
+ wq := work.NewQueue("general", 3)
sq := email.NewServiceQueue(e.Logger, esvc, eq, &mailChecker)
srv := server.New(e, db, config).
Initialize().
M config.example.ini => config.example.ini +6 -0
@@ 182,6 182,12 @@ rate-limit-burst=40
# How long (in minutes) of inactivity does the limit record live for
rate-limit-expire=3
+# How many email queue workers. Defaults to 512
+email-queue-size = 512
+
+# How many general queue workers. Defaults to 512
+general-queue-size = 512
+
[stripe]
secret-key=
public-key=
M go.mod => go.mod +2 -2
@@ 4,7 4,7 @@ go 1.22.5
require (
git.sr.ht/~emersion/gqlclient v0.0.0-20230820050442-8873fe0204b9
- git.sr.ht/~sircmpwn/dowork v0.0.0-20221010085743-46c4299d76a1
+ git.sr.ht/~sircmpwn/dowork v0.0.0-20241219115748-a3928eb084db
github.com/99designs/gqlgen v0.17.57
github.com/Masterminds/squirrel v1.5.4
github.com/alexedwards/scs/postgresstore v0.0.0-20240316134038-7e11d57e8885
@@ 28,7 28,7 @@ require (
golang.org/x/text v0.20.0
golang.org/x/time v0.5.0
hg.code.netlandish.com/~netlandish/sendygo v0.0.0-20230124192435-bbf347776232
- netlandish.com/x/gobwebs v0.0.0-20240828125517-6fc18f8bc093
+ netlandish.com/x/gobwebs v0.0.0-20241219173153-b82c01404952
netlandish.com/x/gobwebs-formguard v0.0.0-20240917220134-4b6f2ec7d1f5
netlandish.com/x/gobwebs-graphql v0.0.0-20241126234432-2cc59b2f7ebd
netlandish.com/x/gobwebs-oauth2 v0.0.0-20240216010400-f0f42e2e3a7c
M go.sum => go.sum +4 -4
@@ 65,8 65,8 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
git.sr.ht/~emersion/gqlclient v0.0.0-20230820050442-8873fe0204b9 h1:QNwHP6WknvS7X6MEFxCpefQb1QJMqgIIt+vn/PVoMMg=
git.sr.ht/~emersion/gqlclient v0.0.0-20230820050442-8873fe0204b9/go.mod h1:kvl/JK0Z3VRmtbBxdOJR4ydyXVouUIcFIXgv4H6rVAY=
-git.sr.ht/~sircmpwn/dowork v0.0.0-20221010085743-46c4299d76a1 h1:EvPKkneKkF/f7zEgKPqIZVyj3jWO8zSmsBOvMhAGqMA=
-git.sr.ht/~sircmpwn/dowork v0.0.0-20221010085743-46c4299d76a1/go.mod h1:8neHEO3503w/rNtttnR0JFpQgM/GFhaafVwvkPsFIDw=
+git.sr.ht/~sircmpwn/dowork v0.0.0-20241219115748-a3928eb084db h1:NX+qg2ZkefnAWL1TvdgumYBggN2f2zYGYQJxIaqaQ5s=
+git.sr.ht/~sircmpwn/dowork v0.0.0-20241219115748-a3928eb084db/go.mod h1:8neHEO3503w/rNtttnR0JFpQgM/GFhaafVwvkPsFIDw=
git.sr.ht/~sircmpwn/getopt v0.0.0-20191230200459-23622cc906b3/go.mod h1:wMEGFFFNuPos7vHmWXfszqImLppbc0wEhh6JBfJIUgw=
git.sr.ht/~sircmpwn/go-bare v0.0.0-20210406120253-ab86bc2846d9 h1:Ahny8Ud1LjVMMAlt8utUFKhhxJtwBAualvsbc/Sk7cE=
git.sr.ht/~sircmpwn/go-bare v0.0.0-20210406120253-ab86bc2846d9/go.mod h1:BVJwbDfVjCjoFiKrhkei6NdGcZYpkDkdyCdg1ukytRA=
@@ 2584,8 2584,8 @@ modernc.org/z v1.0.1-0.20210308123920-1f282aa71362/go.mod h1:8/SRk5C/HgiQWCgXdfp
modernc.org/z v1.0.1/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA=
modernc.org/z v1.2.20/go.mod h1:zU9FiF4PbHdOTUxw+IF8j7ArBMRPsHgq10uVPt6xTzo=
modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4=
-netlandish.com/x/gobwebs v0.0.0-20240828125517-6fc18f8bc093 h1:iUxk5pguGAV6d9C5HXj0/MoEI4gfrrwmqkaRMDIgX9Y=
-netlandish.com/x/gobwebs v0.0.0-20240828125517-6fc18f8bc093/go.mod h1:9uw5D9T02tPpMMr58nThmVp8ZcdrRIAq/SAHsOIim5g=
+netlandish.com/x/gobwebs v0.0.0-20241219173153-b82c01404952 h1:DtzTNLe5MtlYVZuOT0zANWFddsRzM1np4FuMkA/Y0tU=
+netlandish.com/x/gobwebs v0.0.0-20241219173153-b82c01404952/go.mod h1:Q/e3xSbcMqYMpYS360pz5gAA0/cCRrBxNHnCAYODHoA=
netlandish.com/x/gobwebs-formguard v0.0.0-20240917220134-4b6f2ec7d1f5 h1:z5hzVB0R+k2+bh8yuM7XeojGesI2i5tnQHrBNzxG2Xw=
netlandish.com/x/gobwebs-formguard v0.0.0-20240917220134-4b6f2ec7d1f5/go.mod h1:mmpJMveuFlwAj3URXlNmVox3WYCn4OVlozFQdd95wH8=
netlandish.com/x/gobwebs-graphql v0.0.0-20241126234432-2cc59b2f7ebd h1:QJzQj/+2XAEHM0mCxqYleV0sVM2tNZBS6zP6uuOB+Tk=