Fix graceful shutdown

This commit is contained in:
Neil Alexander 2022-04-27 15:29:49 +01:00
parent 103795d33a
commit 923f789ca3
No known key found for this signature in database
GPG key ID: A02A2019A2BB0944
4 changed files with 33 additions and 21 deletions

View file

@ -469,14 +469,14 @@ func (b *BaseDendrite) SetupAndServeHTTP(
}
minwinsvc.SetOnExit(b.ProcessContext.ShutdownDendrite)
b.WaitForShutdown()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_ = internalServ.Shutdown(ctx)
_ = externalServ.Shutdown(ctx)
<-b.ProcessContext.WaitForShutdown()
logrus.Infof("Stopping HTTP listeners")
_ = internalServ.Shutdown(context.Background())
_ = externalServ.Shutdown(context.Background())
logrus.Infof("Stopped HTTP listeners")
b.WaitForShutdown()
}
func (b *BaseDendrite) WaitForShutdown() {

View file

@ -35,6 +35,16 @@ func JetStreamConsumer(
}
go func() {
for {
// If the parent context has given up then there's no point in
// carrying on doing anything, so stop the listener.
select {
case <-ctx.Done():
if err := sub.Unsubscribe(); err != nil {
logrus.WithContext(ctx).Warnf("Failed to unsubscribe %q", durable)
}
return
default:
}
// The context behaviour here is surprising — we supply a context
// so that we can interrupt the fetch if we want, but NATS will still
// enforce its own deadline (roughly 5 seconds by default). Therefore
@ -65,18 +75,18 @@ func JetStreamConsumer(
continue
}
msg := msgs[0]
if err = msg.InProgress(); err != nil {
if err = msg.InProgress(nats.Context(ctx)); err != nil {
logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.InProgress: %w", err))
sentry.CaptureException(err)
continue
}
if f(ctx, msg) {
if err = msg.AckSync(); err != nil {
if err = msg.AckSync(nats.Context(ctx)); err != nil {
logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.AckSync: %w", err))
sentry.CaptureException(err)
}
} else {
if err = msg.Nak(); err != nil {
if err = msg.Nak(nats.Context(ctx)); err != nil {
logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.Nak: %w", err))
sentry.CaptureException(err)
}