multi: track LiT and LND status

With this commit, we now also track the start-up state of LiT and LiT's
connection to LND. These differ from the other sub-servers (loop, pool &
faraday) because a failure to start LiT or LND is fatal and so should
stop the rest of the start-process, however, we still want the webserver
to continue serving the new Status server so that the UI can query the
start-up status of LND and LiT. So: if any errors occur while
starting/connecting to LND or any other errors occur while starting any
of LiTs other processes, then we throw an error but we dont kill the
main LiT process. The main process is only killed upon receiving a
shutdown signal.

Author: ellemouton

rpc_proxy.go

@@ -424,10 +424,16 @@ func (p *rpcProxy) checkSubSystemStarted(requestURI string) error {
 		system = subServerName
 
 	case p.permsMgr.IsLndURI(requestURI):
-		return nil
+		system = LNDSubServer
 
 	case p.permsMgr.IsLitURI(requestURI):
-		return nil
+		system = LitSubServer
+
+		// If the request is for the status server, then we allow the
+		// request even if Lit has not properly started.
+		if isStatusReq(requestURI) {
+			return nil
+		}
 
 	default:
 		return fmt.Errorf("unknown gRPC web request: %v", requestURI)

terminal.go

@@ -237,10 +237,49 @@ func (g *LightningTerminal) Run() error {
 		return fmt.Errorf("error starting UI HTTP server: %v", err)
 	}
 
+	// Attempt to start Lit and all of its sub-servers. If an error is
+	// returned, it means that either one of Lit's internal sub-servers
+	// could not start or LND could not start or be connected to.
+	startErr := g.start()
+	if startErr != nil {
+		g.statusServer.setServerErrored(
+			LitSubServer, "could not start Lit: %v", startErr,
+		)
+	}
+
+	// Now block until we receive an error or the main shutdown
+	// signal.
+	<-shutdownInterceptor.ShutdownChannel()
+	log.Infof("Shutdown signal received")
+
+	if g.rpcProxy != nil {
+		if err := g.rpcProxy.Stop(); err != nil {
+			log.Errorf("Error stopping rpc proxy: %v", err)
+		}
+	}
+
+	if g.httpServer != nil {
+		if err := g.httpServer.Close(); err != nil {
+			log.Errorf("Error stopping UI server: %v", err)
+		}
+	}
+
+	g.wg.Wait()
+
+	return startErr
+}
+
+// start attempts to start all the various components of Litd. Only Litd and
+// LND errors are considered fatal and will result in an error being returned.
+// If any of the sub-servers managed by the subServerMgr error while starting
+// up, these are considered non-fatal and will not result in an error being
+// returned.
+func (g *LightningTerminal) start() error {
 	// Create the instances of our subservers now so we can hook them up to
 	// lnd once it's fully running.
 	g.initSubServers()
 
+	var err error
 	g.accountService, err = accounts.NewService(
 		filepath.Dir(g.cfg.MacaroonPath), g.errQueue.ChanIn(),
 	)
@@ -328,9 +367,7 @@ func (g *LightningTerminal) Run() error {
 		go func() {
 			defer g.wg.Done()
 
-			err := lnd.Main(
-				g.cfg.Lnd, lisCfg, implCfg, shutdownInterceptor,
-			)
+			err := lnd.Main(g.cfg.Lnd, lisCfg, implCfg, interceptor)
 			if e, ok := err.(*flags.Error); err != nil &&
 				(!ok || e.Type != flags.ErrHelp) {
 
@@ -361,19 +398,25 @@ func (g *LightningTerminal) Run() error {
 	case <-readyChan:
 
 	case err := <-g.errQueue.ChanOut():
-		return err
+		g.statusServer.setServerErrored(
+			LNDSubServer, "error from errQueue channel",
+		)
+		return fmt.Errorf("could not start LND: %v", err)
 
 	case <-lndQuit:
-		return nil
+		g.statusServer.setServerErrored(
+			LNDSubServer, "lndQuit channel closed",
+		)
+		return fmt.Errorf("LND has stopped")
 
-	case <-shutdownInterceptor.ShutdownChannel():
-		return errors.New("shutting down")
+	case <-interceptor.ShutdownChannel():
+		return fmt.Errorf("received the shutdown signal")
 	}
 
 	// We now know that starting lnd was successful. If we now run into an
 	// error, we must shut down lnd correctly.
 	defer func() {
-		err := g.shutdown()
+		err := g.shutdownSubServers()
 		if err != nil {
 			log.Errorf("Error shutting down: %v", err)
 		}
@@ -384,6 +427,12 @@ func (g *LightningTerminal) Run() error {
 	if err != nil {
 		return fmt.Errorf("could not connect to LND: %v", err)
 	}
+	if err != nil {
+		g.statusServer.setServerErrored(
+			LNDSubServer, "could not connect to LND: %v", err,
+		)
+		return fmt.Errorf("could not connect to LND")
+	}
 
 	// Initialise any connections to sub-servers that we are running in
 	// remote mode.
@@ -410,12 +459,18 @@ func (g *LightningTerminal) Run() error {
 		return err
 
 	case <-lndQuit:
-		return nil
+		g.statusServer.setServerErrored(
+			LNDSubServer, "lndQuit channel closed",
+		)
+		return fmt.Errorf("LND is not running")
 
-	case <-shutdownInterceptor.ShutdownChannel():
+	case <-interceptor.ShutdownChannel():
 		return errors.New("shutting down")
 	}
 
+	// We can now set the status of LND as running.
+	g.statusServer.setServerRunning(LNDSubServer)
+
 	// If we're in integrated mode, we'll need to wait for lnd to send the
 	// macaroon after unlock before going any further.
 	if g.cfg.LndMode == ModeIntegrated {
@@ -426,8 +481,11 @@ func (g *LightningTerminal) Run() error {
 	// Set up all the LND clients required by LiT.
 	err = g.setUpLNDClients()
 	if err != nil {
-		log.Errorf("Could not set up LND clients: %w", err)
-		return err
+		g.statusServer.setServerErrored(
+			LNDSubServer, "could not set up LND clients: %v", err,
+		)
+
+		return fmt.Errorf("could not start LND")
 	}
 
 	// If we're in integrated and stateless init mode, we won't create
@@ -454,22 +512,28 @@ func (g *LightningTerminal) Run() error {
 		return fmt.Errorf("could not start litd sub-servers: %v", err)
 	}
 
+	// We can now set the status of LiT as running.
+	g.statusServer.setServerRunning(LitSubServer)
+
 	// Now block until we receive an error or the main shutdown signal.
 	select {
 	case err := <-g.errQueue.ChanOut():
 		if err != nil {
-			log.Errorf("Received critical error from subsystem, "+
-				"shutting down: %v", err)
+			return fmt.Errorf("received critical error from "+
+				"subsystem, shutting down: %v", err,
+			)
 		}
 
 	case <-lndQuit:
-		return nil
+		g.statusServer.setServerErrored(
+			LNDSubServer, "lndQuit channel closed",
+		)
+		return fmt.Errorf("LND is not running")
 
-	case <-shutdownInterceptor.ShutdownChannel():
-		log.Infof("Shutdown signal received")
+	case <-interceptor.ShutdownChannel():
 	}
 
-	return nil
+	return fmt.Errorf("received the shutdown signal")
 }
 
 // initSubServers registers the faraday, loop and pool sub-servers with the
@@ -947,8 +1011,9 @@ func (g *LightningTerminal) BuildWalletConfig(ctx context.Context,
 	)
 }
 
-// shutdown stops all subservers that were started and attached to lnd.
-func (g *LightningTerminal) shutdown() error {
+// shutdownSubServers stops all subservers that were started and attached to
+// lnd.
+func (g *LightningTerminal) shutdownSubServers() error {
 	var returnErr error
 
 	err := g.subServerMgr.Stop()
@@ -989,32 +1054,13 @@ func (g *LightningTerminal) shutdown() error {
 		g.restCancel()
 	}
 
-	if g.rpcProxy != nil {
-		if err := g.rpcProxy.Stop(); err != nil {
-			log.Errorf("Error stopping lnd proxy: %v", err)
-			returnErr = err
-		}
-	}
-
 	if g.lndConn != nil {
 		if err := g.lndConn.Close(); err != nil {
 			log.Errorf("Error closing lnd connection: %v", err)
 			returnErr = err
 		}
 	}
 
-	if g.httpServer != nil {
-		if err := g.httpServer.Close(); err != nil {
-			log.Errorf("Error stopping UI server: %v", err)
-			returnErr = err
-		}
-	}
-
-	// In case the error wasn't thrown by lnd, make sure we stop it too.
-	interceptor.RequestShutdown()
-
-	g.wg.Wait()
-
 	// Do we have any last errors to display? We use an anonymous function,
 	// so we can use return instead of breaking to a label in the default
 	// case.