Skip to content

Commit a4ffeb8

Browse files
iQQBotroboquat
iQQBot
authored and
roboquat
committed
[openvsx-proxy] support dynamic upstream url
1 parent 0f0d1bb commit a4ffeb8

File tree

26 files changed

+211
-46
lines changed

26 files changed

+211
-46
lines changed

‎components/common-go/experiments/configcat.go‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,18 @@ package experiments
66

77
import (
88
"context"
9+
"time"
10+
911
configcat "github.com/configcat/go-sdk/v7"
1012
"github.com/gitpod-io/gitpod/common-go/log"
1113
"github.com/sirupsen/logrus"
12-
"time"
1314
)
1415

1516
const (
16-
projectIDAttribute = "project_id"
17-
teamIDAttribute = "team_id"
18-
teamNameAttribute = "team_name"
17+
projectIDAttribute = "project_id"
18+
teamIDAttribute = "team_id"
19+
teamNameAttribute = "team_name"
20+
vscodeClientIDAttribute = "vscode_client_id"
1921
)
2022

2123
func newConfigCatClient(sdkKey string) *configCatClient {
@@ -51,7 +53,7 @@ func (c *configCatClient) GetStringValue(_ context.Context, experimentName strin
5153
return c.client.GetStringValue(experimentName, defaultValue, attributesToUser(attributes))
5254
}
5355

54-
func attributesToUser(attributes Attributes) configcat.UserData {
56+
func attributesToUser(attributes Attributes) *configcat.UserData {
5557
custom := make(map[string]string)
5658

5759
if attributes.TeamID != "" {
@@ -66,7 +68,11 @@ func attributesToUser(attributes Attributes) configcat.UserData {
6668
custom[projectIDAttribute] = attributes.ProjectID
6769
}
6870

69-
return configcat.UserData{
71+
if attributes.VSCodeClientID != "" {
72+
custom[vscodeClientIDAttribute] = attributes.VSCodeClientID
73+
}
74+
75+
return &configcat.UserData{
7076
Identifier: attributes.UserID,
7177
Email: attributes.UserEmail,
7278
Country: "",

‎components/common-go/experiments/types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ type Attributes struct {
2222
ProjectID string
2323
TeamID string
2424
TeamName string
25+
26+
// this is vscode header `x-market-client-id`
27+
VSCodeClientID string
2528
}
2629

2730
// NewClient constructs a new experiments.Client. This is NOT A SINGLETON.

‎components/openvsx-proxy/go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ require (
2929

3030
require (
3131
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
32+
github.com/blang/semver v3.5.1+incompatible // indirect
33+
github.com/configcat/go-sdk/v7 v7.6.0 // indirect
3234
github.com/onsi/ginkgo v1.15.0 // indirect
3335
github.com/onsi/gomega v1.10.5 // indirect
3436
)

‎components/openvsx-proxy/go.sum‎

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎components/openvsx-proxy/pkg/config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ type Config struct {
2424
MaxIdleConnsPerHost int `json:"max_idle_conns_per_host"`
2525
RedisAddr string `json:"redis_addr"`
2626
PrometheusAddr string `json:"prometheusAddr"`
27+
AllowCacheDomain []string `json:"allow_cache_domain"`
2728
}
2829

2930
// Validate validates the configuration to catch issues during startup and not at runtime

‎components/openvsx-proxy/pkg/errorhandler.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@ import (
1414

1515
func (o *OpenVSXProxy) ErrorHandler(rw http.ResponseWriter, r *http.Request, e error) {
1616
reqid := r.Context().Value(REQUEST_ID_CTX).(string)
17-
key := r.Context().Value(REQUEST_CACHE_KEY_CTX).(string)
17+
key, ok := r.Context().Value(REQUEST_CACHE_KEY_CTX).(string)
18+
if !ok {
19+
return
20+
}
1821

1922
logFields := logrus.Fields{
2023
LOG_FIELD_FUNC: "error_handler",

‎components/openvsx-proxy/pkg/handler.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,17 @@ func (o *OpenVSXProxy) Handler(p *httputil.ReverseProxy) func(http.ResponseWrite
4646
log.WithFields(logFields).Debug("handling request")
4747
r = r.WithContext(context.WithValue(r.Context(), REQUEST_ID_CTX, reqid))
4848

49+
upstream := o.GetUpstreamUrl(r)
50+
r = r.WithContext(context.WithValue(r.Context(), UPSTREAM_CTX, upstream))
51+
52+
if o.IsDisabledCache(upstream) {
53+
log.WithFields(logFields).WithField("upstream", upstream.String()).Debug("go without cache")
54+
p.ServeHTTP(rw, r)
55+
o.finishLog(logFields, start, false, false)
56+
o.metrics.DurationRequestProcessingHistogram.Observe(time.Since(start).Seconds())
57+
return
58+
}
59+
4960
key, err := o.key(r)
5061
if err != nil {
5162
log.WithFields(logFields).WithError(err).Error("cannot create cache key")

‎components/openvsx-proxy/pkg/modifyresponse.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,10 @@ import (
1919

2020
func (o *OpenVSXProxy) ModifyResponse(r *http.Response) error {
2121
reqid := r.Request.Context().Value(REQUEST_ID_CTX).(string)
22-
key := r.Request.Context().Value(REQUEST_CACHE_KEY_CTX).(string)
22+
key, ok := r.Request.Context().Value(REQUEST_CACHE_KEY_CTX).(string)
23+
if !ok {
24+
return nil
25+
}
2326

2427
logFields := logrus.Fields{
2528
LOG_FIELD_FUNC: "response_handler",

‎components/openvsx-proxy/pkg/openvsxproxy_test.go‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,22 @@ import (
1111
"net/http"
1212
"net/http/httptest"
1313
"net/http/httputil"
14+
"net/url"
1415
"testing"
1516
)
1617

17-
func createFrontend(backendURL string) (*httptest.Server, *OpenVSXProxy) {
18+
func createFrontend(backendURL string, isDisabledCache bool) (*httptest.Server, *OpenVSXProxy) {
19+
u, _ := url.Parse(backendURL)
1820
cfg := &Config{
1921
URLUpstream: backendURL,
2022
}
23+
if !isDisabledCache {
24+
cfg.AllowCacheDomain = []string{u.Host}
25+
}
2126
openVSXProxy := &OpenVSXProxy{Config: cfg}
2227
openVSXProxy.Setup()
2328

24-
proxy := httputil.NewSingleHostReverseProxy(openVSXProxy.upstreamURL)
29+
proxy := httputil.NewSingleHostReverseProxy(openVSXProxy.defaultUpstreamURL)
2530
proxy.ModifyResponse = openVSXProxy.ModifyResponse
2631
handler := http.HandlerFunc(openVSXProxy.Handler(proxy))
2732
frontend := httptest.NewServer(handler)
@@ -37,7 +42,7 @@ func TestAddResponseToCache(t *testing.T) {
3742
}))
3843
defer backend.Close()
3944

40-
frontend, openVSXProxy := createFrontend(backend.URL)
45+
frontend, openVSXProxy := createFrontend(backend.URL, false)
4146
defer frontend.Close()
4247

4348
frontendClient := frontend.Client()
@@ -64,7 +69,7 @@ func TestServeFromCacheOnUpstreamError(t *testing.T) {
6469
}))
6570
defer backend.Close()
6671

67-
frontend, openVSXProxy := createFrontend(backend.URL)
72+
frontend, openVSXProxy := createFrontend(backend.URL, false)
6873
defer frontend.Close()
6974

7075
requestBody := "Request Body Foo"
@@ -100,3 +105,42 @@ func TestServeFromCacheOnUpstreamError(t *testing.T) {
100105
t.Errorf("got header '%s'; expected '%s'", h, "Foo Bar")
101106
}
102107
}
108+
109+
func TestServeFromNonCacheOnUpstreamError(t *testing.T) {
110+
backend := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
111+
rw.WriteHeader(http.StatusInternalServerError)
112+
}))
113+
defer backend.Close()
114+
115+
frontend, openVSXProxy := createFrontend(backend.URL, true)
116+
defer frontend.Close()
117+
118+
requestBody := "Request Body Foo"
119+
key := fmt.Sprintf("POST / %d %s", len(requestBody), openVSXProxy.hash([]byte(requestBody)))
120+
121+
expectedHeader := make(map[string][]string)
122+
expectedResponse := ""
123+
expectedStatus := 500
124+
125+
openVSXProxy.StoreCache(key, &CacheObject{
126+
Header: expectedHeader,
127+
Body: []byte(expectedResponse),
128+
StatusCode: expectedStatus,
129+
})
130+
131+
frontendClient := frontend.Client()
132+
133+
req, _ := http.NewRequest("POST", frontend.URL, bytes.NewBuffer([]byte(requestBody)))
134+
req.Close = true
135+
res, err := frontendClient.Do(req)
136+
if err != nil {
137+
t.Fatal(err)
138+
}
139+
140+
if res.StatusCode != expectedStatus {
141+
t.Errorf("got status %d; expected %d", res.StatusCode, expectedStatus)
142+
}
143+
if bodyBytes, _ := io.ReadAll(res.Body); string(bodyBytes) != expectedResponse {
144+
t.Errorf("got body '%s'; expected '%s'", string(bodyBytes), expectedResponse)
145+
}
146+
}

‎components/openvsx-proxy/pkg/run.go‎

Lines changed: 42 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"time"
1414

1515
"github.com/eko/gocache/cache"
16+
"github.com/gitpod-io/gitpod/common-go/experiments"
1617
"github.com/gitpod-io/gitpod/common-go/log"
1718
"github.com/sirupsen/logrus"
1819
"golang.org/x/xerrors"
@@ -21,20 +22,49 @@ import (
2122
const (
2223
REQUEST_CACHE_KEY_CTX = "gitpod-cache-key"
2324
REQUEST_ID_CTX = "gitpod-request-id"
25+
UPSTREAM_CTX = "gitpod-upstream"
2426
LOG_FIELD_REQUEST_ID = "request_id"
2527
LOG_FIELD_REQUEST = "request"
2628
LOG_FIELD_FUNC = "func"
2729
LOG_FIELD_STATUS = "status"
2830
)
2931

3032
type OpenVSXProxy struct {
31-
Config *Config
32-
upstreamURL *url.URL
33-
cacheManager *cache.Cache
34-
metrics *Prometheus
33+
Config *Config
34+
defaultUpstreamURL *url.URL
35+
cacheManager *cache.Cache
36+
metrics *Prometheus
37+
experiments experiments.Client
38+
}
39+
40+
func (o *OpenVSXProxy) GetUpstreamUrl(r *http.Request) *url.URL {
41+
reqid := r.Context().Value(REQUEST_ID_CTX).(string)
42+
43+
clientID := r.Header.Get("x-market-client-id")
44+
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
45+
defer cancel()
46+
upstream := o.experiments.GetStringValue(ctx, "openvsx_proxy_upstream", o.Config.URLUpstream, experiments.Attributes{
47+
UserID: reqid,
48+
VSCodeClientID: clientID,
49+
})
50+
upstreamUrl, err := url.Parse(upstream)
51+
if err != nil {
52+
return o.defaultUpstreamURL
53+
}
54+
return upstreamUrl
55+
}
56+
57+
func (o *OpenVSXProxy) IsDisabledCache(u *url.URL) bool {
58+
for _, v := range o.Config.AllowCacheDomain {
59+
if strings.ToLower(u.Host) == v {
60+
return false
61+
}
62+
}
63+
return true
3564
}
3665

3766
func (o *OpenVSXProxy) Setup() error {
67+
o.experiments = experiments.NewClient()
3868
o.metrics = &Prometheus{}
3969
o.metrics.Start(o.Config)
4070

@@ -43,7 +73,7 @@ func (o *OpenVSXProxy) Setup() error {
4373
return xerrors.Errorf("error setting up cache: %v", err)
4474
}
4575

46-
o.upstreamURL, err = url.Parse(o.Config.URLUpstream)
76+
o.defaultUpstreamURL, err = url.Parse(o.Config.URLUpstream)
4777
if err != nil {
4878
return xerrors.Errorf("error parsing upstream URL: %v", err)
4979
}
@@ -54,12 +84,12 @@ func (o *OpenVSXProxy) Setup() error {
5484
}
5585

5686
func (o *OpenVSXProxy) Start() (shutdown func(context.Context) error, err error) {
57-
if o.upstreamURL == nil {
87+
if o.defaultUpstreamURL == nil {
5888
if err := o.Setup(); err != nil {
5989
return nil, err
6090
}
6191
}
62-
proxy := newSingleHostReverseProxy(o.upstreamURL)
92+
proxy := newSingleHostReverseProxy()
6393
proxy.ErrorHandler = o.ErrorHandler
6494
proxy.ModifyResponse = o.ModifyResponse
6595
proxy.Transport = &DurationTrackingTransport{o: o}
@@ -94,7 +124,7 @@ type DurationTrackingTransport struct {
94124

95125
func (t *DurationTrackingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
96126
reqid := r.Context().Value(REQUEST_ID_CTX).(string)
97-
key := r.Context().Value(REQUEST_CACHE_KEY_CTX).(string)
127+
key, _ := r.Context().Value(REQUEST_CACHE_KEY_CTX).(string)
98128

99129
logFields := logrus.Fields{
100130
LOG_FIELD_FUNC: "transport_roundtrip",
@@ -149,9 +179,11 @@ func joinURLPath(a, b *url.URL) (path, rawpath string) {
149179
return a.Path + b.Path, apath + bpath
150180
}
151181

152-
func newSingleHostReverseProxy(target *url.URL) *httputil.ReverseProxy {
153-
targetQuery := target.RawQuery
182+
func newSingleHostReverseProxy() *httputil.ReverseProxy {
154183
director := func(req *http.Request) {
184+
target := req.Context().Value(UPSTREAM_CTX).(*url.URL)
185+
targetQuery := target.RawQuery
186+
155187
originalHost := req.Host
156188

157189
req.URL.Scheme = target.Scheme

0 commit comments

Comments
 (0)