Skip to content

Commit fbc420b

Browse files
kylos101kylos101
authored
[loadgen] fix application of runtime stress & add network stress (#16542)
* [loadgen] remove pvc loadgen configs * [loadgen] add network stress & do network limiting * [loadgen] add supporting fields This makes it easier to find results and related workspaces while testing "live" * [loadgen] fix application of pressure at runtime The underlying workspace image does not have `stress-ng` or `fio` installed. * [loadgen] describe the wait timeout values
1 parent 1d7c01e commit fbc420b

File tree

5 files changed

+29
-185
lines changed

5 files changed

+29
-185
lines changed

‎dev/loadgen/configs/prod-benchmark-pvc.yaml‎

Lines changed: 0 additions & 91 deletions
This file was deleted.

‎dev/loadgen/configs/prod-benchmark.yaml‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,20 @@
33

44
workspaces: 100
55
ideImage: eu.gcr.io/gitpod-core-dev/build/ide/code:commit-ff263e14024f00d0ed78386b4417dfa6bcd4ae2f
6-
waitForRunning: "600s"
7-
waitForStopping: "600s"
6+
# wait 30s longer than the CPU and memory timeouts
7+
waitForRunning: "630s"
8+
# wait 30s longer than the CPU and memory timeouts
9+
waitForStopping: "630s"
810
successRate: 1
911
environment:
1012
- name: "THEIA_SUPERVISOR_TOKENS"
1113
value: '[{"token":"foobar","host":"gitpod-staging.com","scope":["function:getWorkspace","function:getLoggedInUser","function:getPortAuthenticationToken","function:getWorkspaceOwner","function:getWorkspaceUsers","function:isWorkspaceOwner","function:controlAdmission","function:setWorkspaceTimeout","function:getWorkspaceTimeout","function:sendHeartBeat","function:getOpenPorts","function:openPort","function:closePort","function:generateNewGitpodToken","function:takeSnapshot","function:stopWorkspace","resource:workspace::fa498dcc-0a84-448f-9666-79f297ad821a::get/update","resource:workspaceInstance::e0a17083-6a78-441a-9b97-ef90d6aff463::get/update/delete","resource:snapshot::*::create/get","resource:gitpodToken::*::create","resource:userStorage::*::create/get/update"],"expiryDate":"2020-12-01T07:55:12.501Z","reuse":2}]'
1214
workspaceClass: "g1-standard"
1315
workspaceTimeout: 1h
16+
featureFlags:
17+
# https://github.com/gitpod-io/gitpod/blob/df3e15a1d401f525766db1ee5b22203562fe915a/components/ws-manager-api/core.proto#L641
18+
# from core.proto: WORKSPACE_CONNECTION_LIMITING = 10;
19+
- 10
1420
repos:
1521
- cloneURL: https://github.com/gitpod-io/workspace-stress
1622
cloneTarget: main
@@ -61,9 +67,12 @@ repos:
6167
# backup
6268
- name: "BACKUP_SIZE" # size of the backup in gigabyte
6369
value: 12
70+
# network
71+
- name: "NETWORK_WORKERS" # number of workers stressin
72+
value: 1
6473
# tasks
6574
- name: "GITPOD_TASKS"
66-
value: "[{\"name\":\"start cpu stress\",\"command\":\"stress-ng --cpu ${CPU_COUNT:-3} --backoff ${CPU_BACKOFF:-10000000} --timeout ${CPU_TIMEOUT:-600s}\"},{\"name\":\"start io stress\",\"command\":\"fio --name io-stress --eta-newline=5s --filename=/workspace/gitpod.temp --rw=${DISK_IO_MODE} --size=${DISK_IO_FILE_SIZE:-2g} --io_size=${DISK_IO_TOTAL:-50g} --blocksize=${DISK_IO_BLOCKSIZE} --ioengine=libaio --fsync=${DISK_IO_FSYNC} --iodepth=${DISK_IO_DEPTH} --direct=1 --numjobs=${DISK_IO_JOBS} --runtime=${DISK_IO_TIMEOUT:-600}\"},{\"name\":\"start memory stress\",\"command\":\"stress-ng --vm 1 --vm-keep --vm-bytes ${MEMORY_BYTES:-6G} --timeout ${MEMORY_TIMEOUT:-600s}\"},{\"name\":\"create backup file\",\"command\":\"dd if=/dev/zero of=/workspace/benchmark-backup bs=1000M count=${BACKUP_SIZE:-2}\"}]"
75+
value: "[{\"name\":\"Setup\",\"init\":\"sudo install-packages stress-ng fio && gp sync-done setup\"},{\"name\":\"start cpu stress\",\"init\":\"gp sync-await setup\",\"command\":\"stress-ng --cpu ${CPU_COUNT:-3} --backoff ${CPU_BACKOFF:-10000000} --timeout ${CPU_TIMEOUT:-600s}\"},{\"name\":\"start io stress\",\"init\":\"gp sync-await setup\",\"command\":\"fio --name io-stress --eta-newline=5s --filename=/workspace/gitpod.temp --rw=${DISK_IO_MODE} --size=${DISK_IO_FILE_SIZE:-2g} --io_size=${DISK_IO_TOTAL:-50g} --blocksize=${DISK_IO_BLOCKSIZE} --ioengine=libaio --fsync=${DISK_IO_FSYNC} --iodepth=${DISK_IO_DEPTH} --direct=1 --numjobs=${DISK_IO_JOBS} --runtime=${DISK_IO_TIMEOUT:-600}\"},{\"name\":\"start memory stress\",\"init\":\"gp sync-await setup\",\"command\":\"stress-ng --vm 1 --vm-keep --vm-bytes ${MEMORY_BYTES:-6G} --timeout ${MEMORY_TIMEOUT:-600s}\"},{\"name\":\"create backup file\",\"init\":\"gp sync-await setup\",\"command\":\"dd if=/dev/zero of=/workspace/benchmark-backup bs=1000M count=${BACKUP_SIZE:-2}\"},{\"name\":\"start network stress\",\"init\":\"gp sync-await setup\",\"command\":\"stress-ng --class network --all ${NETWORK_WORKERS:-4}\"}]"
6776
- cloneURL: https://github.com/gitpod-io/template-typescript-node
6877
cloneTarget: master
6978
score: 20

‎dev/loadgen/configs/workspace-preview-benchmark-pvc.yaml‎

Lines changed: 0 additions & 83 deletions
This file was deleted.

‎dev/loadgen/configs/workspace-preview-benchmark.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
workspaces: 12
22
ideImage: eu.gcr.io/gitpod-core-dev/build/ide/code:commit-ff263e14024f00d0ed78386b4417dfa6bcd4ae2f
3-
waitForRunning: "600s"
4-
waitForStopping: "600s"
3+
waitForRunning: "630s"
4+
waitForStopping: "630s"
55
successRate: 1
66
environment:
77
- name: "THEIA_SUPERVISOR_TOKENS"
@@ -58,9 +58,12 @@ repos:
5858
# backup
5959
- name: "BACKUP_SIZE" # size of the backup in gigabyte
6060
value: 12
61+
# network
62+
- name: "NETWORK_WORKERS" # number of workers stressin
63+
value: 1
6164
# tasks
6265
- name: "GITPOD_TASKS"
63-
value: "[{\"name\":\"start cpu stress\",\"command\":\"stress-ng --cpu ${CPU_COUNT:-3} --backoff ${CPU_BACKOFF:-10000000} --timeout ${CPU_TIMEOUT:-600s}\"},{\"name\":\"start io stress\",\"command\":\"fio --name io-stress --eta-newline=5s --filename=/workspace/gitpod.temp --rw=${DISK_IO_MODE} --size=${DISK_IO_FILE_SIZE:-2g} --io_size=${DISK_IO_TOTAL:-50g} --blocksize=${DISK_IO_BLOCKSIZE} --ioengine=libaio --fsync=${DISK_IO_FSYNC} --iodepth=${DISK_IO_DEPTH} --direct=1 --numjobs=${DISK_IO_JOBS} --runtime=${DISK_IO_TIMEOUT:-600}\"},{\"name\":\"start memory stress\",\"command\":\"stress-ng --vm 1 --vm-keep --vm-bytes ${MEMORY_BYTES:-6G} --timeout ${MEMORY_TIMEOUT:-600s}\"},{\"name\":\"create backup file\",\"command\":\"dd if=/dev/zero of=/workspace/benchmark-backup bs=1000M count=${BACKUP_SIZE:-2}\"}]"
66+
value: "[{\"name\":\"Setup\",\"init\":\"sudo install-packages stress-ng fio && gp sync-done setup\"},{\"name\":\"start cpu stress\",\"init\":\"gp sync-await setup\",\"command\":\"stress-ng --cpu ${CPU_COUNT:-3} --backoff ${CPU_BACKOFF:-10000000} --timeout ${CPU_TIMEOUT:-600s}\"},{\"name\":\"start io stress\",\"init\":\"gp sync-await setup\",\"command\":\"fio --name io-stress --eta-newline=5s --filename=/workspace/gitpod.temp --rw=${DISK_IO_MODE} --size=${DISK_IO_FILE_SIZE:-2g} --io_size=${DISK_IO_TOTAL:-50g} --blocksize=${DISK_IO_BLOCKSIZE} --ioengine=libaio --fsync=${DISK_IO_FSYNC} --iodepth=${DISK_IO_DEPTH} --direct=1 --numjobs=${DISK_IO_JOBS} --runtime=${DISK_IO_TIMEOUT:-600}\"},{\"name\":\"start memory stress\",\"init\":\"gp sync-await setup\",\"command\":\"stress-ng --vm 1 --vm-keep --vm-bytes ${MEMORY_BYTES:-6G} --timeout ${MEMORY_TIMEOUT:-600s}\"},{\"name\":\"create backup file\",\"init\":\"gp sync-await setup\",\"command\":\"dd if=/dev/zero of=/workspace/benchmark-backup bs=1000M count=${BACKUP_SIZE:-2}\"},{\"name\":\"start network stress\",\"init\":\"gp sync-await setup\",\"command\":\"stress-ng --class network --all ${NETWORK_WORKERS:-4}\"}]"
6467
- cloneURL: https://github.com/gitpod-io/template-typescript-node
6568
cloneTarget: master
6669
score: 20

‎dev/loadgen/pkg/observer/stats.go‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ type StatsSample struct {
3030
Phase api.WorkspacePhase
3131
Failed bool
3232
StartDuration time.Duration
33+
CloneURL string
34+
WorkspaceImage string
35+
WorkspaceName string
3336
}
3437

3538
// NewStatsObserver produces a stats collecting observer
@@ -61,11 +64,14 @@ func NewStatsObserver(cb func(*Stats)) chan<- *loadgen.SessionEvent {
6164
status = make(map[string]*StatsSample)
6265
case loadgen.SessionWorkspaceStart:
6366
status[evt.WorkspaceStart.Spec.Id] = &StatsSample{
64-
InstanceID: evt.WorkspaceStart.Spec.Id,
65-
Start: evt.WorkspaceStart.Time,
66-
Failed: false,
67-
Phase: api.WorkspacePhase_UNKNOWN,
68-
StartDuration: evt.WorkspaceStart.CallDuration,
67+
InstanceID: evt.WorkspaceStart.Spec.Id,
68+
Start: evt.WorkspaceStart.Time,
69+
Failed: false,
70+
Phase: api.WorkspacePhase_UNKNOWN,
71+
StartDuration: evt.WorkspaceStart.CallDuration,
72+
CloneURL: evt.WorkspaceStart.Spec.Metadata.Annotations["context-url"],
73+
WorkspaceImage: evt.WorkspaceStart.Spec.Spec.WorkspaceImage,
74+
WorkspaceName: evt.WorkspaceStart.Spec.Metadata.MetaId,
6975
}
7076
case loadgen.SessionWorkspaceUpdate:
7177
up := evt.WorkspaceUpdate.Update

0 commit comments

Comments
 (0)