Skip to content

Commit 853f111

Browse files
hdurand0710hdurand0710
committed
BUG/MEDIUM: retry opening the runtime socket and allow delay start at startup
1 parent 9649a57 commit 853f111

File tree

6 files changed

+129
-38
lines changed

6 files changed

+129
-38
lines changed

‎client-native/cn.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config
7676
var runtimeClient runtime_api.Runtime
7777

7878
_, globalConf, err := confClient.GetGlobalConfiguration("")
79+
waitForRuntimeOption := runtime_options.AllowDelayedStart(haproxyOptions.DelayedStartMax, haproxyOptions.DelayedStartTick)
7980

8081
// First try to setup master runtime socket
8182
if err == nil {
@@ -87,15 +88,15 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config
8788
if globalConf.Nbproc > 0 {
8889
nbproc := int(globalConf.Nbproc)
8990
ms := runtime_options.MasterSocket(masterSocket, nbproc)
90-
runtimeClient, err = runtime_api.New(ctx, mapsDir, ms)
91+
runtimeClient, err = runtime_api.New(ctx, mapsDir, ms, waitForRuntimeOption)
9192
if err == nil {
9293
return runtimeClient
9394
}
9495
log.Warningf("Error setting up runtime client with master socket: %s : %s", masterSocket, err.Error())
9596
} else {
9697
// if nbproc is not set, use master socket with 1 process
9798
ms := runtime_options.MasterSocket(masterSocket, 1)
98-
runtimeClient, err = runtime_api.New(ctx, mapsDir, ms)
99+
runtimeClient, err = runtime_api.New(ctx, mapsDir, ms, waitForRuntimeOption)
99100
if err == nil {
100101
return runtimeClient
101102
}
@@ -110,7 +111,7 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config
110111
if misc.IsUnixSocketAddr(*r.Address) {
111112
sockets[1] = *r.Address
112113
socketsL := runtime_options.Sockets(sockets)
113-
runtimeClient, err = runtime_api.New(ctx, mapsDir, socketsL)
114+
runtimeClient, err = runtime_api.New(ctx, mapsDir, socketsL, waitForRuntimeOption)
114115
if err == nil {
115116
muSocketsList.Lock()
116117
socketsList = sockets
@@ -143,7 +144,7 @@ func ConfigureRuntimeClient(ctx context.Context, confClient configuration.Config
143144
}
144145

145146
socketLst := runtime_options.Sockets(sockets)
146-
runtimeClient, err = runtime_api.New(ctx, mapsDir, socketLst)
147+
runtimeClient, err = runtime_api.New(ctx, mapsDir, socketLst, waitForRuntimeOption)
147148
if err == nil {
148149
return runtimeClient
149150
}

‎configuration/configuration.go

Lines changed: 37 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"path/filepath"
2525
"strings"
2626
"sync"
27+
"time"
2728

2829
petname "github.com/dustinkirkland/golang-petname"
2930
"github.com/haproxytech/client-native/v5/models"
@@ -34,40 +35,42 @@ import (
3435
var cfg *Configuration
3536

3637
type HAProxyConfiguration struct {
37-
SpoeDir string `long:"spoe-dir" description:"Path to SPOE directory." default:"/etc/haproxy/spoe" group:"resources"`
38-
ServiceName string `long:"service" description:"Name of the HAProxy service" group:"reload"`
39-
HAProxy string `short:"b" long:"haproxy-bin" description:"Path to the haproxy binary file" default:"haproxy" group:"haproxy"`
40-
UserListFile string `long:"userlist-file" description:"Path to the dataplaneapi userlist file. By default userlist is read from HAProxy conf. When specified userlist would be read from this file" group:"userlist"`
41-
ReloadCmd string `short:"r" long:"reload-cmd" description:"Reload command" group:"reload"`
42-
RestartCmd string `short:"s" long:"restart-cmd" description:"Restart command" group:"reload"`
43-
StatusCmd string `long:"status-cmd" description:"Status command" group:"reload"`
44-
NodeIDFile string `long:"fid" description:"Path to file that will dataplaneapi use to write its id (not a pid) that was given to him after joining a cluster" group:"haproxy"`
45-
PIDFile string `long:"pid-file" description:"Path to file that will dataplaneapi use to write its pid" group:"dataplaneapi" example:"/tmp/dataplane.pid"`
46-
ReloadStrategy string `long:"reload-strategy" description:"Either systemd, s6 or custom" default:"custom" group:"reload"`
47-
TransactionDir string `short:"t" long:"transaction-dir" description:"Path to the transaction directory" default:"/tmp/haproxy" group:"transaction"`
48-
ValidateCmd string `long:"validate-cmd" description:"Executes a custom command to perform the HAProxy configuration check" group:"reload"`
49-
BackupsDir string `long:"backups-dir" description:"Path to directory in which to place backup files" group:"transaction"`
50-
MapsDir string `short:"p" long:"maps-dir" description:"Path to directory of map files managed by dataplane" default:"/etc/haproxy/maps" group:"resources"`
51-
SpoeTransactionDir string `long:"spoe-transaction-dir" description:"Path to the SPOE transaction directory" default:"/tmp/spoe-haproxy" group:"resources"`
52-
DataplaneConfig string `short:"f" description:"Path to the dataplane configuration file" default:"/etc/haproxy/dataplaneapi.yaml" yaml:"-"`
53-
ConfigFile string `short:"c" long:"config-file" description:"Path to the haproxy configuration file" default:"/etc/haproxy/haproxy.cfg" group:"haproxy"`
54-
Userlist string `short:"u" long:"userlist" description:"Userlist in HAProxy configuration to use for API Basic Authentication" default:"controller" group:"userlist"`
55-
MasterRuntime string `short:"m" long:"master-runtime" description:"Path to the master Runtime API socket" group:"haproxy"`
56-
SSLCertsDir string `long:"ssl-certs-dir" description:"Path to SSL certificates directory" default:"/etc/haproxy/ssl" group:"resources"`
57-
GeneralStorageDir string `long:"general-storage-dir" description:"Path to general storage directory" default:"/etc/haproxy/general" group:"resources"`
58-
ClusterTLSCertDir string `long:"cluster-tls-dir" description:"Path where cluster tls certificates will be stored. Defaults to same directory as dataplane configuration file" group:"cluster"`
59-
UpdateMapFilesPeriod int64 `long:"update-map-files-period" description:"Elapsed time in seconds between two maps syncing operations" default:"10" group:"resources"`
60-
ReloadDelay int `short:"d" long:"reload-delay" description:"Minimum delay between two reloads (in s)" default:"5" group:"reload"`
61-
MaxOpenTransactions int64 `long:"max-open-transactions" description:"Limit for active transaction in pending state" default:"20" group:"transaction"`
62-
BackupsNumber int `short:"n" long:"backups-number" description:"Number of backup configuration files you want to keep, stored in the config dir with version number suffix" default:"0" group:"transaction"`
63-
ReloadRetention int `long:"reload-retention" description:"Reload retention in days, every older reload id will be deleted" default:"1" group:"reload"`
64-
UID int `long:"uid" description:"User id value to set on start" group:"dataplaneapi" example:"1000"`
65-
GID int `long:"gid" description:"Group id value to set on start" group:"dataplaneapi" example:"1000"`
66-
UpdateMapFiles bool `long:"update-map-files" description:"Flag used for syncing map files with runtime maps values" group:"resources"`
67-
ShowSystemInfo bool `short:"i" long:"show-system-info" description:"Show system info on info endpoint" group:"dataplaneapi"`
68-
MasterWorkerMode bool `long:"master-worker-mode" description:"Flag to enable helpers when running within HAProxy" group:"haproxy"`
69-
DisableInotify bool `long:"disable-inotify" description:"Disables inotify watcher for the configuration file" group:"dataplaneapi"`
70-
DebugSocketPath string `long:"debug-socket-path" description:"Unix socket path for the debugging command socket" group:"dataplaneapi"`
38+
SpoeDir string `long:"spoe-dir" description:"Path to SPOE directory." default:"/etc/haproxy/spoe" group:"resources"`
39+
ServiceName string `long:"service" description:"Name of the HAProxy service" group:"reload"`
40+
HAProxy string `short:"b" long:"haproxy-bin" description:"Path to the haproxy binary file" default:"haproxy" group:"haproxy"`
41+
UserListFile string `long:"userlist-file" description:"Path to the dataplaneapi userlist file. By default userlist is read from HAProxy conf. When specified userlist would be read from this file" group:"userlist"`
42+
ReloadCmd string `short:"r" long:"reload-cmd" description:"Reload command" group:"reload"`
43+
RestartCmd string `short:"s" long:"restart-cmd" description:"Restart command" group:"reload"`
44+
StatusCmd string `long:"status-cmd" description:"Status command" group:"reload"`
45+
NodeIDFile string `long:"fid" description:"Path to file that will dataplaneapi use to write its id (not a pid) that was given to him after joining a cluster" group:"haproxy"`
46+
PIDFile string `long:"pid-file" description:"Path to file that will dataplaneapi use to write its pid" group:"dataplaneapi" example:"/tmp/dataplane.pid"`
47+
ReloadStrategy string `long:"reload-strategy" description:"Either systemd, s6 or custom" default:"custom" group:"reload"`
48+
TransactionDir string `short:"t" long:"transaction-dir" description:"Path to the transaction directory" default:"/tmp/haproxy" group:"transaction"`
49+
ValidateCmd string `long:"validate-cmd" description:"Executes a custom command to perform the HAProxy configuration check" group:"reload"`
50+
BackupsDir string `long:"backups-dir" description:"Path to directory in which to place backup files" group:"transaction"`
51+
MapsDir string `short:"p" long:"maps-dir" description:"Path to directory of map files managed by dataplane" default:"/etc/haproxy/maps" group:"resources"`
52+
SpoeTransactionDir string `long:"spoe-transaction-dir" description:"Path to the SPOE transaction directory" default:"/tmp/spoe-haproxy" group:"resources"`
53+
DataplaneConfig string `short:"f" description:"Path to the dataplane configuration file" default:"/etc/haproxy/dataplaneapi.yaml" yaml:"-"`
54+
ConfigFile string `short:"c" long:"config-file" description:"Path to the haproxy configuration file" default:"/etc/haproxy/haproxy.cfg" group:"haproxy"`
55+
Userlist string `short:"u" long:"userlist" description:"Userlist in HAProxy configuration to use for API Basic Authentication" default:"controller" group:"userlist"`
56+
MasterRuntime string `short:"m" long:"master-runtime" description:"Path to the master Runtime API socket" group:"haproxy"`
57+
SSLCertsDir string `long:"ssl-certs-dir" description:"Path to SSL certificates directory" default:"/etc/haproxy/ssl" group:"resources"`
58+
GeneralStorageDir string `long:"general-storage-dir" description:"Path to general storage directory" default:"/etc/haproxy/general" group:"resources"`
59+
ClusterTLSCertDir string `long:"cluster-tls-dir" description:"Path where cluster tls certificates will be stored. Defaults to same directory as dataplane configuration file" group:"cluster"`
60+
UpdateMapFilesPeriod int64 `long:"update-map-files-period" description:"Elapsed time in seconds between two maps syncing operations" default:"10" group:"resources"`
61+
ReloadDelay int `short:"d" long:"reload-delay" description:"Minimum delay between two reloads (in s)" default:"5" group:"reload"`
62+
MaxOpenTransactions int64 `long:"max-open-transactions" description:"Limit for active transaction in pending state" default:"20" group:"transaction"`
63+
BackupsNumber int `short:"n" long:"backups-number" description:"Number of backup configuration files you want to keep, stored in the config dir with version number suffix" default:"0" group:"transaction"`
64+
ReloadRetention int `long:"reload-retention" description:"Reload retention in days, every older reload id will be deleted" default:"1" group:"reload"`
65+
UID int `long:"uid" description:"User id value to set on start" group:"dataplaneapi" example:"1000"`
66+
GID int `long:"gid" description:"Group id value to set on start" group:"dataplaneapi" example:"1000"`
67+
UpdateMapFiles bool `long:"update-map-files" description:"Flag used for syncing map files with runtime maps values" group:"resources"`
68+
ShowSystemInfo bool `short:"i" long:"show-system-info" description:"Show system info on info endpoint" group:"dataplaneapi"`
69+
MasterWorkerMode bool `long:"master-worker-mode" description:"Flag to enable helpers when running within HAProxy" group:"haproxy"`
70+
DisableInotify bool `long:"disable-inotify" description:"Disables inotify watcher for the configuration file" group:"dataplaneapi"`
71+
DebugSocketPath string `long:"debug-socket-path" description:"Unix socket path for the debugging command socket" group:"dataplaneapi"`
72+
DelayedStartMax time.Duration `long:"delayed-start-max" description:"Maximum duration to wait for the haproxy runtime socket to be ready" default:"30s" group:"haproxy"`
73+
DelayedStartTick time.Duration `long:"delayed-start-tick" description:"Duration between checks for the haproxy runtime socket to be ready" default:"500ms" group:"haproxy"`
7174
}
7275

7376
type User struct {

‎configuration/configuration_storage.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
package configuration
1717

1818
import (
19+
"time"
20+
1921
"github.com/haproxytech/client-native/v5/models"
2022
"github.com/jessevdk/go-flags"
2123

@@ -74,6 +76,8 @@ type configTypeHaproxy struct {
7476
NodeIDFile *string `yaml:"fid,omitempty"`
7577
MasterWorkerMode *bool `yaml:"master_worker_mode,omitempty"`
7678
Reload *configTypeReload `yaml:"reload,omitempty"`
79+
DelayedStartMax *string `yaml:"delayed_start_max,omitempty"`
80+
DelayedStartTick *string `yaml:"delayed_start_tick,omitempty"`
7781
}
7882

7983
type configTypeUserlist struct {
@@ -391,6 +395,16 @@ func copyToConfiguration(cfg *Configuration) { //nolint:cyclop,maintidx
391395
if cfgStorage.LogTargets != nil {
392396
cfg.LogTargets = *cfgStorage.LogTargets
393397
}
398+
if cfgStorage.Dataplaneapi != nil && cfgStorage.Haproxy.DelayedStartMax != nil && !misc.HasOSArg("", "delayed-start-max", "") {
399+
if d, err := time.ParseDuration(*cfgStorage.Haproxy.DelayedStartMax); err == nil {
400+
cfg.HAProxy.DelayedStartMax = d
401+
}
402+
}
403+
if cfgStorage.Dataplaneapi != nil && cfgStorage.Haproxy.DelayedStartTick != nil && !misc.HasOSArg("", "delayed-start-tick", "") {
404+
if d, err := time.ParseDuration(*cfgStorage.Haproxy.DelayedStartTick); err == nil {
405+
cfg.HAProxy.DelayedStartTick = d
406+
}
407+
}
394408
}
395409

396410
func copyConfigurationToStorage(cfg *Configuration) {

‎configuration/examples/example-full.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ haproxy:
5959
master_runtime: null # string
6060
fid: null # string
6161
master_worker_mode: false # bool
62+
delayed_start_max: 30s # time.Duration
63+
delayed_start_tick: 500ms # time.Duration
6264
reload:
6365
reload_delay: 5 # int 2
6466
reload_cmd: "systemctl reload haproxy"

‎configure_data_plane.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ import (
5858
"github.com/haproxytech/dataplaneapi/operations/specification"
5959
"github.com/haproxytech/dataplaneapi/operations/specification_openapiv3"
6060
"github.com/haproxytech/dataplaneapi/rate"
61+
"github.com/haproxytech/dataplaneapi/resilient"
6162
socket_runtime "github.com/haproxytech/dataplaneapi/runtime"
6263

6364
// import various crypting algorithms
@@ -290,6 +291,7 @@ func configureAPI(api *operations.DataPlaneAPI) http.Handler { //nolint:cyclop,m
290291
})
291292

292293
// setup transaction handlers
294+
client = resilient.NewClient(client)
293295
api.TransactionsStartTransactionHandler = &handlers.StartTransactionHandlerImpl{Client: client}
294296
api.TransactionsDeleteTransactionHandler = &handlers.DeleteTransactionHandlerImpl{Client: client}
295297
api.TransactionsGetTransactionHandler = &handlers.GetTransactionHandlerImpl{Client: client}

‎resilient/client.go

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// Copyright 2019 HAProxy Technologies
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
//
15+
16+
package resilient
17+
18+
import (
19+
"context"
20+
"errors"
21+
22+
client_native "github.com/haproxytech/client-native/v5"
23+
"github.com/haproxytech/client-native/v5/runtime"
24+
cn "github.com/haproxytech/dataplaneapi/client-native"
25+
dataplaneapi_config "github.com/haproxytech/dataplaneapi/configuration"
26+
)
27+
28+
type Client struct {
29+
client_native.HAProxyClient
30+
}
31+
32+
func NewClient(c client_native.HAProxyClient) *Client {
33+
return &Client{
34+
c,
35+
}
36+
}
37+
38+
// Runtime is a wrapper around HAProxyClient.Runtime
39+
// that retries once to configure the runtime client if it failed
40+
func (c *Client) Runtime() (runtime.Runtime, error) {
41+
runtime, err := c.HAProxyClient.Runtime()
42+
43+
// We already have a valid runtime
44+
// Let's return it
45+
if err == nil {
46+
return runtime, nil
47+
}
48+
49+
// Now, for let's try to reconfigure once the runtime
50+
cfg, err := c.HAProxyClient.Configuration()
51+
if err != nil {
52+
return nil, err
53+
}
54+
55+
dpapiCfg := dataplaneapi_config.Get()
56+
haproxyOptions := dpapiCfg.HAProxy
57+
58+
// Let's disable the delayed start by putting a max value to 0
59+
// This is important to not block the handlers by waiting the DelayedStartMax that we wait for when we start
60+
haproxyOptions.DelayedStartMax = 0
61+
// let's retry
62+
rnt := cn.ConfigureRuntimeClient(context.Background(), cfg, haproxyOptions)
63+
if rnt == nil {
64+
return nil, errors.New("retry - unable to configure runtime client")
65+
}
66+
c.HAProxyClient.ReplaceRuntime(rnt)
67+
68+
return c.HAProxyClient.Runtime()
69+
}

0 commit comments

Comments
 (0)