From c3e67261b58f12343b0f600333b2c279c5225faa Mon Sep 17 00:00:00 2001 From: Michael Tautschnig Date: Thu, 12 Oct 2017 12:29:08 +0000 Subject: [PATCH] Script to automate performance evaluation of CBMC on AWS Evaluation is performed for a chosen GitHub or CodeCommit repository and commit/branch/tag by running (a subset of) SV-COMP benchmarks. Runs are done both in optimised as well as in profiling mode. Execution should be as simple as ./perf_test.py \ -r https://github.com/diffblue/cbmc -c develop \ -e tautschn@amazon.com assuming that AWS access keys are set up (as required for AWS cli use) and boto3 is installed. Emails will then be sent as results become available. The script sets up (and persists) an S3 bucket for storing results, SNS queues for email updates on the process, as well as an EBS snapshot containing the benmarking data. For each benchmarking run, builds are set up and performed via CodeBuild. Evaluation is then performed using AutoScalingGroups synchronised via SQS. The design premise for this work was: "Compare multiple configurations for performance, correctness, capabilities." This was broken down into: Configuration: target platform, timeout, memory limit, benchmark set, tool options, source version (=repository + revision). Compare: log files, counterexamples, CPU profile, memory profile, verification results. --- scripts/perf-test/codebuild.yaml | 127 ++++++ scripts/perf-test/ebs.yaml | 53 +++ scripts/perf-test/ec2.yaml | 394 ++++++++++++++++++ scripts/perf-test/perf_test.py | 669 +++++++++++++++++++++++++++++++ scripts/perf-test/s3.yaml | 38 ++ scripts/perf-test/sns.yaml | 27 ++ scripts/perf-test/sqs.yaml | 27 ++ 7 files changed, 1335 insertions(+) create mode 100644 scripts/perf-test/codebuild.yaml create mode 100644 scripts/perf-test/ebs.yaml create mode 100644 scripts/perf-test/ec2.yaml create mode 100755 scripts/perf-test/perf_test.py create mode 100644 scripts/perf-test/s3.yaml create mode 100644 scripts/perf-test/sns.yaml create mode 100644 scripts/perf-test/sqs.yaml diff --git a/scripts/perf-test/codebuild.yaml b/scripts/perf-test/codebuild.yaml new file mode 100644 index 00000000000..45663349b61 --- /dev/null +++ b/scripts/perf-test/codebuild.yaml @@ -0,0 +1,127 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + S3Bucket: + Type: String + + PerfTestId: + Type: String + + RepoType: + Type: String + + Repository: + Type: String + +Resources: + CodeBuildRole: + Type: "AWS::IAM::Role" + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: "sts:AssumeRole" + RoleName: !Sub "CR-${PerfTestId}" + Policies: + - PolicyName: !Sub "CP-${PerfTestId}" + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - "s3:PutObject" + Effect: Allow + Resource: !Join ["/", [!Sub "arn:aws:s3:::${S3Bucket}", "*"]] + - Action: + - "logs:CreateLogGroup" + - "logs:CreateLogStream" + - "logs:PutLogEvents" + Effect: Allow + Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/*' + + ReleaseBuild: + Type: "AWS::CodeBuild::Project" + Properties: + Artifacts: + Type: S3 + Location: !Ref S3Bucket + Path: !Ref PerfTestId + Name: release + Environment: + ComputeType: BUILD_GENERAL1_LARGE + Image: aws/codebuild/ubuntu-base:14.04 + Type: LINUX_CONTAINER + Name: !Sub "perf-test-release-build-${PerfTestId}" + ServiceRole: !Ref CodeBuildRole + Source: + BuildSpec: !Sub | + version: 0.2 + phases: + install: + commands: + - apt-get update -y + - apt-get install -y software-properties-common + - add-apt-repository ppa:ubuntu-toolchain-r/test + - apt-get update -y + - apt-get install -y libwww-perl g++-5 flex bison git + - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 1 + - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 1 + build: + commands: + - echo ${Repository} > COMMIT_INFO + - git rev-parse --short HEAD >> COMMIT_INFO + - git log HEAD^..HEAD >> COMMIT_INFO + - make -C src minisat2-download glucose-download + - make -C src -j8 + artifacts: + files: + - src/cbmc/cbmc + - COMMIT_INFO + discard-paths: yes + Type: !Ref RepoType + Location: !Ref Repository + + ProfilingBuild: + Type: "AWS::CodeBuild::Project" + Properties: + Artifacts: + Type: S3 + Location: !Ref S3Bucket + Path: !Ref PerfTestId + Name: profiling + Environment: + ComputeType: BUILD_GENERAL1_LARGE + Image: aws/codebuild/ubuntu-base:14.04 + Type: LINUX_CONTAINER + Name: !Sub "perf-test-profiling-build-${PerfTestId}" + ServiceRole: !Ref CodeBuildRole + Source: + BuildSpec: !Sub | + version: 0.2 + phases: + install: + commands: + - apt-get update -y + - apt-get install -y software-properties-common + - add-apt-repository ppa:ubuntu-toolchain-r/test + - apt-get update -y + - apt-get install -y libwww-perl g++-5 flex bison git + - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 1 + - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 1 + build: + commands: + - echo ${Repository} > COMMIT_INFO + - git rev-parse --short HEAD >> COMMIT_INFO + - git log HEAD^..HEAD >> COMMIT_INFO + - make -C src minisat2-download glucose-download + - make -C src -j8 CXXFLAGS="-O2 -pg -g -finline-limit=4" LINKFLAGS="-pg" + artifacts: + files: + - src/cbmc/cbmc + - COMMIT_INFO + discard-paths: yes + Type: !Ref RepoType + Location: !Ref Repository diff --git a/scripts/perf-test/ebs.yaml b/scripts/perf-test/ebs.yaml new file mode 100644 index 00000000000..609554c8f39 --- /dev/null +++ b/scripts/perf-test/ebs.yaml @@ -0,0 +1,53 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + Ami: + Type: String + + AvailabilityZone: + Type: String + +Resources: + EC2Instance: + Type: "AWS::EC2::Instance" + Properties: + InstanceType: t2.micro + ImageId: !Ref Ami + AvailabilityZone: !Ref AvailabilityZone + Volumes: + - Device: "/dev/sdf" + VolumeId: !Ref BaseVolume + UserData: !Base64 | + #!/bin/bash + set -e + # wait to make sure volume is available + sleep 10 + mkfs.ext4 /dev/xvdf + mount /dev/xvdf /mnt + apt-get -y update + apt-get install git + cd /mnt + git clone --depth 1 --branch svcomp17 \ + https://github.com/sosy-lab/sv-benchmarks.git + git clone --depth 1 \ + https://github.com/sosy-lab/benchexec.git + git clone --depth 1 --branch trunk \ + https://github.com/sosy-lab/cpachecker.git + git clone --depth 1 \ + https://github.com/diffblue/cprover-sv-comp.git + halt + + BaseVolume: + Type: "AWS::EC2::Volume" + DeletionPolicy: Snapshot + Properties: + AvailabilityZone: !Ref AvailabilityZone + Size: 8 + Tags: + - Key: Name + Value: perf-test-base + +Outputs: + InstanceId: + Value: !Ref EC2Instance diff --git a/scripts/perf-test/ec2.yaml b/scripts/perf-test/ec2.yaml new file mode 100644 index 00000000000..5933777f6e2 --- /dev/null +++ b/scripts/perf-test/ec2.yaml @@ -0,0 +1,394 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + InstanceType: + Type: String + + Ami: + Type: String + + SnapshotId: + Type: String + + AvailabilityZone: + Type: String + + S3Bucket: + Type: String + + PerfTestId: + Type: String + + SnsTopic: + Type: String + + SqsArn: + Type: String + + SqsUrl: + Type: String + + MaxPrice: + Type: String + + FleetSize: + Type: String + + SSHKeyName: + Type: String + +Conditions: + UseSpot: !Not [!Equals [!Ref MaxPrice, ""]] + + UseKey: !Not [!Equals [!Ref SSHKeyName, ""]] + +Resources: + EC2Role: + Type: "AWS::IAM::Role" + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: "sts:AssumeRole" + RoleName: !Sub "ER-${PerfTestId}" + Policies: + - PolicyName: !Sub "EP-${PerfTestId}" + PolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - "s3:PutObject" + - "s3:GetObject" + Effect: Allow + Resource: !Join ["/", [!Sub "arn:aws:s3:::${S3Bucket}", "*"]] + - Action: + - "sns:Publish" + Effect: Allow + Resource: !Ref SnsTopic + - Action: + - "sqs:DeleteMessage" + - "sqs:DeleteQueue" + - "sqs:GetQueueAttributes" + - "sqs:ReceiveMessage" + Effect: Allow + Resource: !Ref SqsArn + - Action: + - "sqs:DeleteMessage" + - "sqs:DeleteQueue" + - "sqs:GetQueueAttributes" + - "sqs:ReceiveMessage" + - "sqs:SendMessage" + Effect: Allow + Resource: !Sub "${SqsArn}-run" + - Action: + - "cloudformation:DeleteStack" + Effect: Allow + Resource: !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/perf-test-*/*" + - Action: + - "autoscaling:DeleteAutoScalingGroup" + - "autoscaling:DeleteLaunchConfiguration" + - "autoscaling:DescribeAutoScalingGroups" + - "autoscaling:DescribeScalingActivities" + - "autoscaling:UpdateAutoScalingGroup" + - "ec2:DeleteSecurityGroup" + - "iam:DeleteInstanceProfile" + - "iam:DeleteRole" + - "iam:DeleteRolePolicy" + - "iam:RemoveRoleFromInstanceProfile" + Effect: Allow + Resource: "*" + + EC2InstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + Roles: + - !Ref EC2Role + + SecurityGroupInSSHWorld: + Type: "AWS::EC2::SecurityGroup" + DependsOn: EC2Role + Properties: + GroupDescription: SSH access + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: '0.0.0.0/0' + + LaunchConfiguration: + Type: "AWS::AutoScaling::LaunchConfiguration" + Properties: + BlockDeviceMappings: + - DeviceName: "/dev/sdf" + Ebs: + DeleteOnTermination: True + SnapshotId: !Ref SnapshotId + VolumeSize: 64 + IamInstanceProfile: !Ref EC2InstanceProfile + ImageId: !Ref Ami + InstanceType: !Ref InstanceType + KeyName: + !If [UseKey, !Ref SSHKeyName, !Ref "AWS::NoValue"] + SecurityGroups: + - !Ref SecurityGroupInSSHWorld + SpotPrice: + !If [UseSpot, !Ref MaxPrice, !Ref "AWS::NoValue"] + UserData: + Fn::Base64: !Sub | + #!/bin/bash + set -x -e + + # wait to make sure volume is available + sleep 10 + e2fsck -f -y /dev/xvdf + resize2fs /dev/xvdf + mount /dev/xvdf /mnt + + # install packages + apt-get -y update + apt-get install -y git time wget binutils awscli make jq + apt-get install -y gcc libc6-dev-i386 + + # cgroup set up for benchexec + chmod o+wt '/sys/fs/cgroup/cpuset/' + chmod o+wt '/sys/fs/cgroup/cpu,cpuacct/user.slice' + chmod o+wt '/sys/fs/cgroup/memory/user.slice' + chmod o+wt '/sys/fs/cgroup/freezer/' + + # AWS Sig-v4 access + aws configure set s3.signature_version s3v4 + + # send instance-terminated message + # http://rogueleaderr.com/post/48795010760/how-to-notifyemail-yourself-when-an-ec2-instance/amp + cat >/etc/init.d/ec2-terminate <<"EOF" + #!/bin/bash + ### BEGIN INIT INFO + # Provides: ec2-terminate + # Required-Start: $network $syslog + # Required-Stop: + # Default-Start: + # Default-Stop: 0 1 6 + # Short-Description: ec2-terminate + # Description: send termination email + ### END INIT INFO + # + + case "$1" in + start|status) + exit 0 + ;; + stop) + # run the below + ;; + *) + exit 1 + ;; + esac + + ut=$(cat /proc/uptime | cut -f1 -d" ") + aws --region us-east-1 sns publish \ + --topic-arn ${SnsTopic} \ + --message "instance terminating after $ut s at ${MaxPrice} USD/h" + sleep 3 # make sure the message has time to send + aws s3 cp /var/log/cloud-init-output.log \ + s3://${S3Bucket}/${PerfTestId}/$HOSTNAME.cloud-init-output.log + + exit 0 + EOF + chmod a+x /etc/init.d/ec2-terminate + update-rc.d ec2-terminate defaults + systemctl start ec2-terminate + + # prepare for tool packaging + cd /mnt + cd cprover-sv-comp + mkdir -p src/cbmc/ + touch LICENSE + cd .. + mkdir -p run + cd run + wget -O cbmc.xml https://raw.githubusercontent.com/sosy-lab/sv-comp/master/benchmark-defs/cbmc.xml + sed -i 's/witness.graphml/${!logfile_path_abs}${!inputfile_name}-witness.graphml/' cbmc.xml + cd .. + mkdir -p tmp + export TMPDIR=/mnt/tmp + + # reduce the likelihood of multiple hosts processing the + # same message (in addition to SQS's message hiding) + sleep $(expr $RANDOM % 30) + retry=1 + + while true + do + sqs=$(aws --region ${AWS::Region} sqs receive-message \ + --queue-url ${SqsUrl} | \ + jq -r '.Messages[0].Body,.Messages[0].ReceiptHandle') + + if [ -z "$sqs" ] + then + # no un-read messages in the input queue; let's look + # at -run + n_msgs=$(aws --region ${AWS::Region} sqs \ + get-queue-attributes \ + --queue-url ${SqsUrl}-run \ + --attribute-names \ + ApproximateNumberOfMessages | \ + jq -r '.Attributes.ApproximateNumberOfMessages') + + if [ $retry -eq 1 ] + then + retry=0 + sleep 30 + continue + elif [ -n "$n_msgs" ] && [ "$n_msgs" = "0" ] + then + # shut down the infrastructure + aws --region us-east-1 sns publish \ + --topic-arn ${SnsTopic} \ + --message "Trying to delete stacks in ${AWS::Region}" + aws --region ${AWS::Region} cloudformation \ + delete-stack --stack-name \ + perf-test-sqs-${PerfTestId} + aws --region ${AWS::Region} cloudformation \ + delete-stack --stack-name \ + perf-test-exec-${PerfTestId} + halt + fi + + # the queue is gone, or other host will be turning + # off the lights + halt + fi + + retry=1 + bm=$(echo $sqs | cut -f1 -d" ") + cfg=$(echo $bm | cut -f1 -d"-") + t=$(echo $bm | cut -f2- -d"-") + msg=$(echo $sqs | cut -f2- -d" ") + + # mark $bm in-progress + aws --region ${AWS::Region} sqs send-message \ + --queue-url ${SqsUrl}-run \ + --message-body $bm-$(hostname) + + # there is no guarantee of cross-queue action ordering + # sleep for a bit to reduce the likelihood of missing + # in-progress messages while the input queue is empty + sleep 3 + + # remove it from the input queue + aws --region ${AWS::Region} sqs delete-message \ + --queue-url ${SqsUrl} \ + --receipt-handle $msg + + cd /mnt/cprover-sv-comp + rm -f src/cbmc/cbmc + aws s3 cp s3://${S3Bucket}/${PerfTestId}/$cfg/cbmc \ + src/cbmc/cbmc + chmod a+x src/cbmc/cbmc + make CBMC=. YEAR=N CBMC-sv-comp-N.tar.gz + cd ../run + tar xzf ../cprover-sv-comp/CBMC-sv-comp-N.tar.gz + rm ../cprover-sv-comp/CBMC-sv-comp-N.tar.gz + + date + echo "Task: $t" + + # compute the number of possible executors + max_par=$(cat /proc/cpuinfo | grep ^processor | wc -l) + mem=$(free -g | grep ^Mem | awk '{print $2}') + if [ $cfg != "profiling" ] + then + mem=$(expr $mem / 15) + else + mem=$(expr $mem / 7) + fi + if [ $mem -lt $max_par ] + then + max_par=$mem + fi + + if [ $cfg != "profiling" ] + then + ../benchexec/bin/benchexec cbmc.xml --no-container \ + --task $t -T 900s -M 15GB -o logs-$t/ \ + -N $max_par -c 1 + if [ -d logs-$t/cbmc.*.logfiles ] + then + cd logs-$t + tar czf witnesses.tar.gz cbmc.*.logfiles + rm -rf cbmc.*.logfiles + cd .. + fi + if [ -f logs-$t/*.xml.bz2 ] + then + start_date="$(echo ${PerfTestId} | cut -f1-3 -d-) $(echo ${PerfTestId} | cut -f4-6 -d- | sed 's/-/:/g')" + cd logs-$t + bunzip2 *.xml.bz2 + perl -p -i -e \ + "s/^(/dev/null 2>&1 + then + gprof --sum ./cbmc-binary *.gmon.out.* + gprof ./cbmc-binary gmon.sum > sum.profile-$t + rm -f gmon.sum gmon.out *.gmon.out.* + aws s3 cp sum.profile-$t \ + s3://${S3Bucket}/${PerfTestId}/$cfg/sum.profile-$t + fi + fi + rm -rf logs-$t sum.profile-$t + date + + # clear out the in-progress message + while true + do + sqs=$(aws --region ${AWS::Region} sqs \ + receive-message \ + --queue-url ${SqsUrl}-run \ + --visibility-timeout 10 | \ + jq -r '.Messages[0].Body,.Messages[0].ReceiptHandle') + bm2=$(echo $sqs | cut -f1 -d" ") + msg2=$(echo $sqs | cut -f2- -d" ") + + if [ "$bm2" = "$bm-$(hostname)" ] + then + aws --region ${AWS::Region} sqs delete-message \ + --queue-url ${SqsUrl}-run \ + --receipt-handle $msg2 + break + fi + done + done + + AutoScalingGroup: + Type: "AWS::AutoScaling::AutoScalingGroup" + Properties: + AvailabilityZones: + - !Ref AvailabilityZone + DesiredCapacity: !Ref FleetSize + LaunchConfigurationName: !Ref LaunchConfiguration + MaxSize: !Ref FleetSize + MinSize: 1 + +Outputs: + ASGId: + Value: !Ref AutoScalingGroup diff --git a/scripts/perf-test/perf_test.py b/scripts/perf-test/perf_test.py new file mode 100755 index 00000000000..0f4de36dc40 --- /dev/null +++ b/scripts/perf-test/perf_test.py @@ -0,0 +1,669 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import argparse +import boto3 +import concurrent.futures +import contextlib +import datetime +import json +import logging +import os +import re +import shutil +import sys +import tempfile +import time +import urllib + + +@contextlib.contextmanager +def make_temp_directory(): + """ + create a temporary directory and remove it once the statement completes + """ + temp_dir = tempfile.mkdtemp() + try: + yield temp_dir + finally: + shutil.rmtree(temp_dir) + + +def same_dir(filename): + d = os.path.dirname(os.path.abspath(__file__)) + return os.path.join(d, filename) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-r', '--repository', type=str, required=True, + help='CodeCommit or GitHub repository containing ' + + 'code to evaluate') + parser.add_argument('-c', '--commit-id', type=str, required=True, + help='git revision to evaluate') + parser.add_argument('-e', '--email', type=str, required=True, + help='Email address to notify about results') + parser.add_argument('-t', '--instance-type', type=str, + default='r4.16xlarge', + help='Amazon EC2 instance type to use ' + + '(default: r4.16xlarge)') + parser.add_argument('-m', '--mode', + choices=['spot', 'on-demand', 'batch'], + default='spot', + help='Choose fleet to run benchmarks on ' + + '(default: Amazon EC2 Spot fleet)') + parser.add_argument('-R', '--region', type=str, + help='Set fixed region instead of cheapest fleet') + parser.add_argument('-j', '--parallel', type=int, default=4, + help='Fleet size of concurrently running hosts ' + + '(default: 4)') + parser.add_argument('-k', '--ssh-key-name', type=str, default='', + help='EC2 key name for SSH access to fleet') + parser.add_argument('-K', '--ssh-key', type=str, + help='SSH public key file for access to fleet ' + + '(requires -K/--ssh-key-name)') + parser.add_argument('-T', '--tasks', type=str, + default='quick', + help='Subset of tasks to run (quick, full; ' + + 'default: quick; or name of SV-COMP task)') + + args = parser.parse_args() + assert(args.repository.startswith('https://github.com/') or + args.repository.startswith('https://git-codecommit.')) + assert(not args.ssh_key or args.ssh_key_name) + if args.ssh_key: + assert(os.path.isfile(args.ssh_key)) + + return args + + +def prepare_s3(session, bucket_name, artifact_uploaded_arn): + # create a bucket for storing artifacts + logger = logging.getLogger('perf_test') + s3 = session.resource('s3', region_name='us-east-1') + buckets = list(s3.buckets.all()) + + for b in buckets: + if b.name == bucket_name: + logger.info('us-east-1: S3 bucket {} exists'.format(bucket_name)) + return + + cfn = session.resource('cloudformation', region_name='us-east-1') + with open(same_dir('s3.yaml')) as f: + CFN_s3 = f.read() + cfn.create_stack( + StackName='perf-test-s3', + TemplateBody=CFN_s3, + Parameters=[ + { + 'ParameterKey': 'SnsTopicArn', + 'ParameterValue': artifact_uploaded_arn + }, + { + 'ParameterKey': 'S3BucketName', + 'ParameterValue': bucket_name + } + ]) + + waiter = cfn.meta.client.get_waiter('stack_create_complete') + waiter.wait(StackName='perf-test-s3', WaiterConfig={'Delay': 10}) + logger.info('us-east-1: S3 bucket {} set up'.format(bucket_name)) + + +def prepare_sns_s3(session, email, bucket_name): + # create instance_terminated topic + # create artifact_uploaded topic + logger = logging.getLogger('perf_test') + sns = session.resource('sns', region_name='us-east-1') + topics = list(sns.topics.all()) + instance_terminated_arn = None + artifact_uploaded_arn = None + + for t in topics: + if t.attributes['DisplayName'] == 'instance_terminated': + instance_terminated_arn = t.arn + logger.info('us-east-1: SNS topic instance_terminated exists') + if int(t.attributes['SubscriptionsPending']) > 0: + logger.warning('us-east-1: SNS topic instance_terminated ' + + 'has pending subscription confirmations') + elif t.attributes['DisplayName'] == 'artifact_uploaded': + artifact_uploaded_arn = t.arn + logger.info('us-east-1: SNS topic artifact_uploaded exists') + if int(t.attributes['SubscriptionsPending']) > 0: + logger.warning('us-east-1: SNS topic artifact_uploaded ' + + 'has pending subscription confirmations') + + cfn = session.resource('cloudformation', region_name='us-east-1') + + with open(same_dir('sns.yaml')) as f: + CFN_sns = f.read() + + if not instance_terminated_arn: + cfn.create_stack( + StackName='perf-test-sns-instance-term', + TemplateBody=CFN_sns, + Parameters=[ + { + 'ParameterKey': 'NotificationAddress', + 'ParameterValue': email + }, + { + 'ParameterKey': 'SnsTopicName', + 'ParameterValue': 'instance_terminated' + } + ]) + + if not artifact_uploaded_arn: + cfn.create_stack( + StackName='perf-test-sns-artifact-uploaded', + TemplateBody=CFN_sns, + Parameters=[ + { + 'ParameterKey': 'NotificationAddress', + 'ParameterValue': email + }, + { + 'ParameterKey': 'SnsTopicName', + 'ParameterValue': 'artifact_uploaded' + } + ]) + + waiter = cfn.meta.client.get_waiter('stack_create_complete') + if not instance_terminated_arn: + waiter.wait( + StackName='perf-test-sns-instance-term', + WaiterConfig={'Delay': 10}) + stack = cfn.Stack('perf-test-sns-instance-term') + instance_terminated_arn = stack.outputs[0]['OutputValue'] + logger.info('us-east-1: SNS topic instance_terminated set up') + if not artifact_uploaded_arn: + waiter.wait( + StackName='perf-test-sns-artifact-uploaded', + WaiterConfig={'Delay': 10}) + stack = cfn.Stack('perf-test-sns-artifact-uploaded') + artifact_uploaded_arn = stack.outputs[0]['OutputValue'] + logger.info('us-east-1: SNS topic artifact_uploaded set up') + + prepare_s3(session, bucket_name, artifact_uploaded_arn) + + return instance_terminated_arn + + +def select_region(session, mode, region, instance_type): + # find the region and az with the lowest spot price for the chosen instance + # type + # based on https://gist.github.com/pahud/fbbc1fd80fac4544fd0a3a480602404e + logger = logging.getLogger('perf_test') + + if not region: + ec2 = session.client('ec2') + regions = [r['RegionName'] for r in ec2.describe_regions()['Regions']] + else: + regions = [region] + + min_region = None + min_az = None + min_price = None + + if mode == 'on-demand': + logger.info('global: Fetching on-demand prices for ' + instance_type) + with make_temp_directory() as tmp_dir: + for r in regions: + json_file = os.path.join(tmp_dir, 'index.json') + urllib.request.urlretrieve( + 'https://pricing.us-east-1.amazonaws.com/offers/' + + 'v1.0/aws/AmazonEC2/current/' + r + '/index.json', + json_file) + with open(json_file) as jf: + json_result = json.load(jf) + key = None + for p in json_result['products']: + v = json_result['products'][p] + a = v['attributes'] + if ((v['productFamily'] == 'Compute Instance') and + (a['instanceType'] == instance_type) and + (a['tenancy'] == 'Shared') and + (a['operatingSystem'] == 'Linux')): + assert(not key) + key = p + for c in json_result['terms']['OnDemand'][key]: + v = json_result['terms']['OnDemand'][key][c] + for p in v['priceDimensions']: + price = v['priceDimensions'][p]['pricePerUnit']['USD'] + if min_region is None or float(price) < min_price: + min_region = r + ec2 = session.client('ec2', region_name=r) + azs = ec2.describe_availability_zones( + Filters=[{'Name': 'region-name', 'Values': [r]}]) + min_az = azs['AvailabilityZones'][0]['ZoneName'] + min_price = float(price) + + logger.info('global: Lowest on-demand price: {} ({}): {}'.format( + min_region, min_az, min_price)) + else: + logger.info('global: Fetching spot prices for ' + instance_type) + for r in regions: + ec2 = session.client('ec2', region_name=r) + res = ec2.describe_spot_price_history( + InstanceTypes=[instance_type], + ProductDescriptions=['Linux/UNIX'], + StartTime=datetime.datetime.now()) + history = res['SpotPriceHistory'] + for az in history: + if min_region is None or float(az['SpotPrice']) < min_price: + min_region = r + min_az = az['AvailabilityZone'] + min_price = float(az['SpotPrice']) + + logger.info('global: Lowest spot price: {} ({}): {}'.format( + min_region, min_az, min_price)) + + # http://aws-ubuntu.herokuapp.com/ + # 20170919 - Ubuntu 16.04 LTS (xenial) - hvm:ebs-ssd + AMI_ids = { + "Mappings": { + "RegionMap": { + "ap-northeast-1": {"64": "ami-8422ebe2"}, + "ap-northeast-2": {"64": "ami-0f6fb461"}, + "ap-south-1": {"64": "ami-08a5e367"}, + "ap-southeast-1": {"64": "ami-e6d3a585"}, + "ap-southeast-2": {"64": "ami-391ff95b"}, + "ca-central-1": {"64": "ami-e59c2581"}, + "eu-central-1": {"64": "ami-5a922335"}, + "eu-west-1": {"64": "ami-17d11e6e"}, + "eu-west-2": {"64": "ami-e1f2e185"}, + "sa-east-1": {"64": "ami-a3e39ecf"}, + "us-east-1": {"64": "ami-d651b8ac"}, + "us-east-2": {"64": "ami-9686a4f3"}, + "us-west-1": {"64": "ami-2d5c6d4d"}, + "us-west-2": {"64": "ami-ecc63a94"} + } + } + } + + ami = AMI_ids['Mappings']['RegionMap'][min_region]['64'] + + return (min_region, min_az, min_price, ami) + + +def prepare_ebs(session, region, az, ami): + # create an ebs volume that contains the benchmark sources + logger = logging.getLogger('perf_test') + ec2 = session.client('ec2', region_name=region) + snapshots = ec2.describe_snapshots( + OwnerIds=['self'], + Filters=[ + { + 'Name': 'tag:Name', + 'Values': ['perf-test-base'] + } + ]) + + if snapshots['Snapshots']: + logger.info(region + ': EBS snapshot exists') + else: + logger.info(region + ': EBS snapshot preparation required') + cfn = session.resource('cloudformation', region_name=region) + with open(same_dir('ebs.yaml')) as f: + CFN_ebs = f.read() + stack = cfn.create_stack( + StackName='perf-test-build-ebs', + TemplateBody=CFN_ebs, + Parameters=[ + { + 'ParameterKey': 'Ami', + 'ParameterValue': ami + }, + { + 'ParameterKey': 'AvailabilityZone', + 'ParameterValue': az + } + ]) + + waiter = cfn.meta.client.get_waiter('stack_create_complete') + waiter.wait(StackName='perf-test-build-ebs') + instance_id = stack.outputs[0]['OutputValue'] + logger.info(region + ': Waiting for EBS snapshot preparation on ' + + instance_id) + waiter = ec2.get_waiter('instance_stopped') + waiter.wait(InstanceIds=[instance_id]) + stack.delete() + waiter = cfn.meta.client.get_waiter('stack_delete_complete') + waiter.wait(StackName='perf-test-build-ebs') + logger.info(region + ': EBS snapshot prepared') + + snapshots = ec2.describe_snapshots( + OwnerIds=['self'], + Filters=[ + { + 'Name': 'tag:Name', + 'Values': ['perf-test-base'] + } + ]) + + return snapshots['Snapshots'][0]['SnapshotId'] + + +def build(session, repository, commit_id, bucket_name, perf_test_id): + # build the chosen commit in CodeBuild + logger = logging.getLogger('perf_test') + + if repository.startswith('https://github.com/'): + repo_type = 'GITHUB' + else: + repo_type = 'CODECOMMIT' + + cfn = session.resource('cloudformation', region_name='us-east-1') + stack_name = 'perf-test-codebuild-' + perf_test_id + with open(same_dir('codebuild.yaml')) as f: + CFN_codebuild = f.read() + stack = cfn.create_stack( + StackName=stack_name, + TemplateBody=CFN_codebuild, + Parameters=[ + { + 'ParameterKey': 'S3Bucket', + 'ParameterValue': bucket_name + }, + { + 'ParameterKey': 'PerfTestId', + 'ParameterValue': perf_test_id + }, + { + 'ParameterKey': 'RepoType', + 'ParameterValue': repo_type + }, + { + 'ParameterKey': 'Repository', + 'ParameterValue': repository + } + ], + Capabilities=['CAPABILITY_NAMED_IAM']) + + waiter = cfn.meta.client.get_waiter('stack_create_complete') + waiter.wait(StackName=stack_name) + logger.info('us-east-1: CodeBuild configuration complete') + + codebuild = session.client('codebuild', region_name='us-east-1') + rel_build = codebuild.start_build( + projectName='perf-test-release-build-' + perf_test_id, + sourceVersion=commit_id)['build']['id'] + prof_build = codebuild.start_build( + projectName='perf-test-profiling-build-' + perf_test_id, + sourceVersion=commit_id)['build']['id'] + + logger.info('us-east-1: Waiting for builds to complete') + all_complete = False + completed = {} + while not all_complete: + time.sleep(10) + response = codebuild.batch_get_builds(ids=[rel_build, prof_build]) + all_complete = True + for b in response['builds']: + if b['buildStatus'] == 'IN_PROGRESS': + all_complete = False + break + elif not completed.get(b['projectName']): + logger.info('us-east-1: Build {} ended: {}'.format( + b['projectName'], b['buildStatus'])) + assert(b['buildStatus'] == 'SUCCEEDED') + completed[b['projectName']] = True + + stack.delete() + waiter = cfn.meta.client.get_waiter('stack_delete_complete') + waiter.wait(StackName=stack_name) + logger.info('us-east-1: CodeBuild complete and stack cleaned') + + +def prepare_sqs(session, region, perf_test_id, tasks): + # create a bucket for storing artifacts + logger = logging.getLogger('perf_test') + + cfn = session.resource('cloudformation', region_name=region) + stack_name = 'perf-test-sqs-' + perf_test_id + with open(same_dir('sqs.yaml')) as f: + CFN_sqs = f.read() + stack = cfn.create_stack( + StackName=stack_name, + TemplateBody=CFN_sqs, + Parameters=[ + { + 'ParameterKey': 'PerfTestId', + 'ParameterValue': perf_test_id + } + ]) + + waiter = cfn.meta.client.get_waiter('stack_create_complete') + waiter.wait(StackName=stack_name, WaiterConfig={'Delay': 10}) + for o in stack.outputs: + if o['OutputKey'] == 'QueueArn': + arn = o['OutputValue'] + elif o['OutputKey'] == 'QueueName': + queue = o['OutputValue'] + elif o['OutputKey'] == 'QueueUrl': + url = o['OutputValue'] + else: + assert(False) + logger.info(region + ': SQS queues {}, {}-run set up'.format( + queue, queue)) + + seed_queue(session, region, queue, tasks) + + return (queue, arn, url) + + +def seed_queue(session, region, queue, task_set): + # set up the tasks + logger = logging.getLogger('perf_test') + + all_tasks = ['ConcurrencySafety-Main', 'DefinedBehavior-Arrays', + 'DefinedBehavior-TerminCrafted', 'MemSafety-Arrays', + 'MemSafety-Heap', 'MemSafety-LinkedLists', + 'MemSafety-Other', 'MemSafety-TerminCrafted', + 'Overflows-BitVectors', 'Overflows-Other', + 'ReachSafety-Arrays', 'ReachSafety-BitVectors', + 'ReachSafety-ControlFlow', 'ReachSafety-ECA', + 'ReachSafety-Floats', 'ReachSafety-Heap', + 'ReachSafety-Loops', 'ReachSafety-ProductLines', + 'ReachSafety-Recursive', 'ReachSafety-Sequentialized', + 'Systems_BusyBox_MemSafety', 'Systems_BusyBox_Overflows', + 'Systems_DeviceDriversLinux64_ReachSafety', + 'Termination-MainControlFlow', 'Termination-MainHeap', + 'Termination-Other'] + + sqs = session.resource('sqs', region_name=region) + queue = sqs.get_queue_by_name(QueueName=queue) + if task_set == 'full': + tasks = all_tasks + elif task_set == 'quick': + tasks = ['ReachSafety-Loops', 'ReachSafety-BitVectors'] + else: + tasks = [task_set] + + for t in tasks: + assert(t in set(all_tasks)) + + for t in tasks: + response = queue.send_messages( + Entries=[ + {'Id': '1', 'MessageBody': 'release-' + t}, + {'Id': '2', 'MessageBody': 'profiling-' + t} + ]) + assert(not response.get('Failed')) + + +def run_perf_test( + session, mode, region, az, ami, instance_type, sqs_arn, sqs_url, + parallel, snapshot_id, instance_terminated_arn, bucket_name, + perf_test_id, price, ssh_key_name): + # create an EC2 instance and trigger benchmarking + logger = logging.getLogger('perf_test') + + if mode == 'spot': + price = str(price*3) + elif mode == 'on-demand': + price = '' + else: + # Batch not yet implemented + assert(False) + + stack_name = 'perf-test-exec-' + perf_test_id + logger.info(region + ': Creating stack ' + stack_name) + cfn = session.resource('cloudformation', region_name=region) + with open(same_dir('ec2.yaml')) as f: + CFN_ec2 = f.read() + stack = cfn.create_stack( + StackName=stack_name, + TemplateBody=CFN_ec2, + Parameters=[ + { + 'ParameterKey': 'Ami', + 'ParameterValue': ami + }, + { + 'ParameterKey': 'AvailabilityZone', + 'ParameterValue': az + }, + { + 'ParameterKey': 'InstanceType', + 'ParameterValue': instance_type + }, + { + 'ParameterKey': 'SnapshotId', + 'ParameterValue': snapshot_id + }, + { + 'ParameterKey': 'S3Bucket', + 'ParameterValue': bucket_name + }, + { + 'ParameterKey': 'PerfTestId', + 'ParameterValue': perf_test_id + }, + { + 'ParameterKey': 'SnsTopic', + 'ParameterValue': instance_terminated_arn + }, + { + 'ParameterKey': 'SqsArn', + 'ParameterValue': sqs_arn + }, + { + 'ParameterKey': 'SqsUrl', + 'ParameterValue': sqs_url + }, + { + 'ParameterKey': 'MaxPrice', + 'ParameterValue': price + }, + { + 'ParameterKey': 'FleetSize', + 'ParameterValue': str(parallel) + }, + { + 'ParameterKey': 'SSHKeyName', + 'ParameterValue': ssh_key_name + } + ], + Capabilities=['CAPABILITY_NAMED_IAM']) + + waiter = cfn.meta.client.get_waiter('stack_create_complete') + waiter.wait(StackName=stack_name) + asg_name = stack.outputs[0]['OutputValue'] + asg = session.client('autoscaling', region_name=region) + # make sure hosts that have been shut down don't come back + asg.suspend_processes( + AutoScalingGroupName=asg_name, + ScalingProcesses=['ReplaceUnhealthy']) + while True: + res = asg.describe_auto_scaling_instances() + if len(res['AutoScalingInstances']) == parallel: + break + logger.info(region + ': Waiting for AutoScalingGroup to be populated') + time.sleep(10) + # https://gist.github.com/alertedsnake/4b85ea44481f518cf157 + instances = [a['InstanceId'] for a in res['AutoScalingInstances'] + if a['AutoScalingGroupName'] == asg_name] + + ec2 = session.client('ec2', region_name=region) + for instance_id in instances: + i_res = ec2.describe_instances(InstanceIds=[instance_id]) + name = i_res['Reservations'][0]['Instances'][0]['PublicDnsName'] + logger.info(region + ': Running benchmarks on ' + name) + + +def main(): + logging_format = "%(asctime)-15s: %(message)s" + logging.basicConfig(format=logging_format) + logger = logging.getLogger('perf_test') + logger.setLevel('DEBUG') + + args = parse_args() + + # pick the most suitable region + session = boto3.session.Session() + (region, az, price, ami) = select_region( + session, args.mode, args.region, args.instance_type) + + # fail early if key configuration would fail + if args.ssh_key_name: + ec2 = session.client('ec2', region_name=region) + res = ec2.describe_key_pairs( + Filters=[ + {'Name': 'key-name', 'Values': [args.ssh_key_name]} + ]) + if not args.ssh_key: + assert(len(res['KeyPairs']) == 1) + elif len(res['KeyPairs']): + logger.warning(region + ': Key pair "' + args.ssh_key_name + + '" already exists, ignoring key material') + else: + with open(args.ssh_key) as kf: + pk = kf.read() + ec2.import_key_pair( + KeyName=args.ssh_key_name, PublicKeyMaterial=pk) + + # build a unique id for this performance test run + perf_test_id = str(datetime.datetime.utcnow().isoformat( + sep='-', timespec='seconds')) + '-' + args.commit_id + perf_test_id = re.sub('[:/_\.\^~ ]', '-', perf_test_id) + logger.info('global: Preparing performance test ' + perf_test_id) + + # target storage name + account_id = session.client('sts').get_caller_identity()['Account'] + bucket_name = "perf-test-" + account_id + + # configuration set, let's create the infrastructure + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as e: + session1 = boto3.session.Session() + sns_s3_future = e.submit( + prepare_sns_s3, session1, args.email, bucket_name) + session2 = boto3.session.Session() + build_future = e.submit( + build, session2, args.repository, args.commit_id, bucket_name, + perf_test_id) + session3 = boto3.session.Session() + ebs_future = e.submit(prepare_ebs, session3, region, az, ami) + session4 = boto3.session.Session() + sqs_future = e.submit( + prepare_sqs, session4, region, perf_test_id, args.tasks) + + # wait for all preparation steps to complete + instance_terminated_arn = sns_s3_future.result() + build_future.result() + snapshot_id = ebs_future.result() + (queue, sqs_arn, sqs_url) = sqs_future.result() + + run_perf_test( + session, args.mode, region, az, ami, args.instance_type, + sqs_arn, sqs_url, args.parallel, snapshot_id, + instance_terminated_arn, bucket_name, perf_test_id, price, + args.ssh_key_name) + + return 0 + + +if __name__ == '__main__': + rc = main() + sys.exit(rc) diff --git a/scripts/perf-test/s3.yaml b/scripts/perf-test/s3.yaml new file mode 100644 index 00000000000..d8c7a8b5a79 --- /dev/null +++ b/scripts/perf-test/s3.yaml @@ -0,0 +1,38 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + SnsTopicArn: + Type: String + + S3BucketName: + Type: String + +Resources: + SnsTopic: + Type: "AWS::SNS::TopicPolicy" + Properties: + Topics: + - !Ref SnsTopicArn + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - sns:Publish + Principal: + AWS: "*" + Resource: !Ref SnsTopicArn + Condition: + ArnLike: + AWS:SourceArn: !Sub "arn:aws:s3:::${S3BucketName}" + + S3Bucket: + DependsOn: SnsTopic + Type: "AWS::S3::Bucket" + Properties: + BucketName: !Ref S3BucketName + NotificationConfiguration: + TopicConfigurations: + - Event: s3:ObjectCreated:* + Topic: !Ref SnsTopicArn diff --git a/scripts/perf-test/sns.yaml b/scripts/perf-test/sns.yaml new file mode 100644 index 00000000000..a51899074c1 --- /dev/null +++ b/scripts/perf-test/sns.yaml @@ -0,0 +1,27 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + NotificationAddress: + Type: String + + SnsTopicName: + Type: String + +Resources: + SnsTopic: + Type: "AWS::SNS::Topic" + Properties: + DisplayName: !Ref SnsTopicName + TopicName: !Ref SnsTopicName + + SnsSubscription: + Type: "AWS::SNS::Subscription" + Properties: + Endpoint: !Ref NotificationAddress + Protocol: email + TopicArn: !Ref SnsTopic + +Outputs: + TopicArn: + Value: !Ref SnsTopic diff --git a/scripts/perf-test/sqs.yaml b/scripts/perf-test/sqs.yaml new file mode 100644 index 00000000000..11d6d87a1da --- /dev/null +++ b/scripts/perf-test/sqs.yaml @@ -0,0 +1,27 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Parameters: + PerfTestId: + Type: String + +Resources: + Queue: + Type: "AWS::SQS::Queue" + Properties: + QueueName: !Sub "perf-test-${PerfTestId}" + + QueueDone: + Type: "AWS::SQS::Queue" + Properties: + QueueName: !Sub "perf-test-${PerfTestId}-run" + +Outputs: + QueueName: + Value: !GetAtt Queue.QueueName + + QueueUrl: + Value: !Ref Queue + + QueueArn: + Value: !GetAtt Queue.Arn