diff --git a/config/headers.go b/config/headers.go
index 7c3149eb3..4a0be4a10 100644
--- a/config/headers.go
+++ b/config/headers.go
@@ -17,6 +17,7 @@
 package config
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"os"
@@ -50,17 +51,22 @@ var reservedHeaders = map[string]struct{}{
 
 // Headers represents the configuration for HTTP headers.
 type Headers struct {
-	Headers map[string]Header `yaml:",inline" json:",inline"`
+	Headers map[string]Header `yaml:",inline"`
 	dir     string
 }
 
-// Headers represents the configuration for HTTP headers.
+// Header represents the configuration for a single HTTP header.
 type Header struct {
 	Values  []string `yaml:"values,omitempty" json:"values,omitempty"`
 	Secrets []Secret `yaml:"secrets,omitempty" json:"secrets,omitempty"`
 	Files   []string `yaml:"files,omitempty" json:"files,omitempty"`
 }
 
+func (h Headers) MarshalJSON() ([]byte, error) {
+	// Inline the Headers map when serializing JSON because json encoder doesn't support "inline" directive.
+	return json.Marshal(h.Headers)
+}
+
 // SetDirectory records the directory to make headers file relative to the
 // configuration file.
 func (h *Headers) SetDirectory(dir string) {
diff --git a/config/http_config_test.go b/config/http_config_test.go
index 67b7408d5..b66a46942 100644
--- a/config/http_config_test.go
+++ b/config/http_config_test.go
@@ -35,6 +35,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
 )
 
@@ -2004,6 +2005,91 @@ func TestMarshalURLWithSecret(t *testing.T) {
 	}
 }
 
+func TestHTTPClientConfig_Marshal(t *testing.T) {
+	proxyURL, err := url.Parse("http://localhost:8080")
+	require.NoError(t, err)
+
+	t.Run("without HTTP headers", func(t *testing.T) {
+		config := &HTTPClientConfig{
+			ProxyConfig: ProxyConfig{
+				ProxyURL: URL{proxyURL},
+			},
+		}
+
+		t.Run("YAML", func(t *testing.T) {
+			actualYAML, err := yaml.Marshal(config)
+			require.NoError(t, err)
+			require.YAMLEq(t, `
+proxy_url: "http://localhost:8080"
+follow_redirects: false
+enable_http2: false
+http_headers: null
+`, string(actualYAML))
+
+			// Unmarshalling the YAML should get the same struct in input.
+			actual := &HTTPClientConfig{}
+			require.NoError(t, yaml.Unmarshal(actualYAML, actual))
+			require.Equal(t, config, actual)
+		})
+
+		t.Run("JSON", func(t *testing.T) {
+			actualJSON, err := json.Marshal(config)
+
+			require.NoError(t, err)
+			require.JSONEq(t, `{
+				"proxy_url":"http://localhost:8080",
+				"tls_config":{"insecure_skip_verify":false},
+				"follow_redirects":false,
+				"enable_http2":false,
+				"http_headers":null
+			}`, string(actualJSON))
+
+			// Unmarshalling the JSON should get the same struct in input.
+			actual := &HTTPClientConfig{}
+			require.NoError(t, json.Unmarshal(actualJSON, actual))
+			require.Equal(t, config, actual)
+		})
+	})
+
+	t.Run("with HTTP headers", func(t *testing.T) {
+		config := &HTTPClientConfig{
+			ProxyConfig: ProxyConfig{
+				ProxyURL: URL{proxyURL},
+			},
+			HTTPHeaders: &Headers{
+				Headers: map[string]Header{
+					"X-Test": {
+						Values: []string{"Value-1", "Value-2"},
+					},
+				},
+			},
+		}
+
+		actualYAML, err := yaml.Marshal(config)
+		require.NoError(t, err)
+		require.YAMLEq(t, `
+proxy_url: "http://localhost:8080"
+follow_redirects: false
+enable_http2: false
+http_headers:
+  X-Test:
+    values:
+    - Value-1
+    - Value-2
+`, string(actualYAML))
+
+		actualJSON, err := json.Marshal(config)
+		require.NoError(t, err)
+		require.JSONEq(t, `{
+			"proxy_url":"http://localhost:8080",
+			"tls_config":{"insecure_skip_verify":false},
+			"follow_redirects":false,
+			"enable_http2":false,
+			"http_headers":{"X-Test":{"values":["Value-1","Value-2"]}}
+		}`, string(actualJSON))
+	})
+}
+
 func TestOAuth2Proxy(t *testing.T) {
 	_, _, err := LoadHTTPConfigFile("testdata/http.conf.oauth2-proxy.good.yml")
 	if err != nil {