Skip to content

Commit 9f4d9d5

Browse files
manuel-alvarez-alvarezmanuel-alvarez-alvarez
authored
Add initial version of tracer benchmarks (DataDog#5604)
* Add initial version of tracer benchmarks * Remove custom user creation for the benchmarks * Update benchmarking-platform branch
1 parent 908ba7a commit 9f4d9d5

19 files changed

Lines changed: 547 additions & 3 deletions

File tree

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,3 +67,7 @@ replay_pid*
6767
############
6868
_circle_ci_cache_*
6969
upstream.env
70+
71+
# Benchmarks #
72+
benchmark/reports
73+
benchmark/tracer

‎.gitlab/benchmarks.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
variables:
2-
BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/relenv-microbenchmarking-platform:dd-trace-java
2+
BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-java-benchmarks
33

44
benchmarks:
55
stage: benchmarks
@@ -11,9 +11,8 @@ benchmarks:
1111
script:
1212
- export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
1313
- git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
14-
- git clone --branch dd-trace-java https://github.com/DataDog/relenv-microbenchmarking-platform /platform && cd /platform
14+
- git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
1515
- ./steps/capture-hardware-software-info.sh
16-
- ./steps/fetch-tracers.sh
1716
- ./steps/run-benchmarks.sh
1817
- ./steps/analyze-results.sh
1918
- ./steps/upload-results-to-s3.sh

‎benchmark/Dockerfile‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# Petclinic download and compilation stage
2+
FROM eclipse-temurin:17-jammy as petclinic
3+
4+
RUN apt-get update \
5+
&& apt-get -y install git \
6+
&& apt-get -y clean \
7+
&& rm -rf /var/lib/apt/lists/*
8+
9+
RUN git clone --depth 1 --branch main --single-branch https://github.com/spring-projects/spring-petclinic.git \
10+
&& cd spring-petclinic \
11+
&& ./mvnw dependency:go-offline
12+
13+
RUN cd spring-petclinic \
14+
&& ./mvnw package -Dmaven.test.skip=true \
15+
&& cp target/*.jar /spring-petclinic.jar
16+
17+
18+
# Insecure bank download and compilation stage
19+
FROM eclipse-temurin:17-jammy as insecure-bank
20+
21+
RUN apt-get update \
22+
&& apt-get -y install git \
23+
&& apt-get -y clean \
24+
&& rm -rf /var/lib/apt/lists/*
25+
26+
RUN git clone --depth 1 --branch malvarez/spring-boot --single-branch https://github.com/hdiv/insecure-bank.git \
27+
&& cd insecure-bank \
28+
&& ./gradlew -q dependencies
29+
30+
RUN cd insecure-bank \
31+
&& ./gradlew bootWar \
32+
&& cp build/libs/*.war /insecure-bank.war
33+
34+
35+
# K6 distribution
36+
FROM grafana/xk6 as k6
37+
38+
USER 0:0
39+
RUN xk6 build \
40+
--with github.com/szkiba/xk6-faker@latest \
41+
--output /k6
42+
43+
44+
# sirun distribution
45+
FROM debian:bookworm-slim as sirun
46+
47+
RUN apt-get update \
48+
&& apt-get -y install wget \
49+
&& apt-get -y clean \
50+
&& rm -rf /var/lib/apt/lists/*
51+
52+
USER 0:0
53+
ARG SIRUN_VERSION=0.1.11
54+
RUN wget -O sirun.tar.gz https://github.com/DataDog/sirun/releases/download/v$SIRUN_VERSION/sirun-v$SIRUN_VERSION-x86_64-unknown-linux-musl.tar.gz \
55+
&& tar -xzf sirun.tar.gz \
56+
&& rm sirun.tar.gz
57+
58+
59+
FROM debian:bookworm-slim
60+
61+
RUN apt-get update \
62+
&& apt-get -y install git curl wget procps gettext-base \
63+
&& apt-get -y clean \
64+
&& rm -rf /var/lib/apt/lists/*
65+
66+
COPY --from=eclipse-temurin:8-jammy /opt/java/openjdk /usr/lib/jvm/8
67+
COPY --from=eclipse-temurin:11-jammy /opt/java/openjdk /usr/lib/jvm/11
68+
COPY --from=eclipse-temurin:17-jammy /opt/java/openjdk /usr/lib/jvm/17
69+
70+
RUN rm -rf \
71+
/usr/lib/jvm/*/man \
72+
/usr/lib/jvm/*/src.zip \
73+
/usr/lib/jvm/*/lib/src.zip \
74+
/usr/lib/jvm/*/demo \
75+
/usr/lib/jvm/*/sample
76+
77+
ENV JAVA_8_HOME=/usr/lib/jvm/8
78+
ENV JAVA_11_HOME=/usr/lib/jvm/11
79+
ENV JAVA_17_HOME=/usr/lib/jvm/17
80+
ENV JAVA_HOME=${JAVA_8_HOME}
81+
ENV PATH=${PATH}:${JAVA_HOME}/bin
82+
83+
COPY --from=k6 /k6 /usr/bin/k6
84+
COPY --from=sirun /sirun /usr/bin/sirun
85+
86+
RUN mkdir -p /app
87+
88+
COPY --from=petclinic /spring-petclinic.jar /app/spring-petclinic.jar
89+
ENV PETCLINIC=/app/spring-petclinic.jar
90+
91+
COPY --from=insecure-bank /insecure-bank.war /app/insecure-bank.war
92+
ENV INSECURE_BANK=/app/insecure-bank.war

‎benchmark/README.MD‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Benchmarks
2+
3+
This directory contains different types of benchmarks.
4+
5+
## Running Benchmarks via Docker
6+
7+
Docker allows the execution of benchmarks without needing to install and configure your development environment. For example, package installation and installation of sirun is performed automatically.
8+
9+
In order to run benchmarks using Docker, issue the following command from the benchmark folder of the project:
10+
11+
```sh
12+
./run.sh
13+
```
14+
15+
Once it finishes, the reports will be available in the reports folder.
16+
17+
### Running specific benchmarks
18+
19+
If you want to run only a specific category of benchmarks you can do it via arguments:
20+
21+
1. Run startup benchmarks
22+
```sh
23+
./run.sh startup [application]?
24+
```
25+
26+
2. Run load benchmarks
27+
```sh
28+
./run.sh load [application]?
29+
```

‎benchmark/benchmarks.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/usr/bin/env bash
2+
set -eu
3+
4+
readonly SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
5+
export TRACER_DIR="${SCRIPT_DIR}/.."
6+
export REPORTS_DIR="${SCRIPT_DIR}/reports"
7+
export UTILS_DIR="${SCRIPT_DIR}/utils"
8+
export SHELL_UTILS_DIR="${UTILS_DIR}/shell"
9+
export K6_UTILS_DIR="${UTILS_DIR}/k6"
10+
export TRACER="${SCRIPT_DIR}/tracer/dd-java-agent.jar"
11+
export NO_AGENT_VARIANT="no_agent"
12+
13+
run_benchmarks() {
14+
local type=$1
15+
if [[ -d "${type}" ]] && [[ -f "${type}/run.sh" ]]; then
16+
cd "${type}"
17+
./run.sh "$@"
18+
cd "${SCRIPT_DIR}"
19+
fi
20+
}
21+
22+
# Find or rebuild tracer to be used in the benchmarks
23+
if [[ ! -f "${TRACER}" ]]; then
24+
mkdir -p "${SCRIPT_DIR}/tracer"
25+
cd "${TRACER_DIR}"
26+
readonly TRACER_VERSION=$(./gradlew properties -q | grep "version:" | awk '{print $2}')
27+
readonly TRACER_COMPILED="${SCRIPT_DIR}/../dd-java-agent/build/libs/dd-java-agent-${TRACER_VERSION}.jar"
28+
if [[ ! -f "${TRACER_COMPILED}" ]]; then
29+
echo "Tracer not found, starting gradle compile ..."
30+
./gradlew assemble
31+
fi
32+
cp "${TRACER_COMPILED}" "${TRACER}"
33+
cd "${SCRIPT_DIR}"
34+
fi
35+
36+
# Cleanup previous reports
37+
rm -rf "${REPORTS_DIR}"
38+
mkdir -p "${REPORTS_DIR}"
39+
40+
if [[ "$#" == '0' ]]; then
41+
for type in 'startup' 'load'; do
42+
run_benchmarks "$type"
43+
done
44+
else
45+
run_benchmarks "$@"
46+
fi

‎benchmark/load/insecure-bank/benchmark.json‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
"name": "load_insecure-bank",
3+
"setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"",
4+
"service": "bash -c \"${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080/login ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"",
5+
"run": "bash -c \"java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log\"",
6+
"timeout": 150,
7+
"iterations": 1,
8+
"variants": {
9+
"${NO_AGENT_VARIANT}": {
10+
"env": {
11+
"VARIANT": "${NO_AGENT_VARIANT}",
12+
"JAVA_OPTS": ""
13+
}
14+
},
15+
"tracing": {
16+
"env": {
17+
"VARIANT": "tracing",
18+
"JAVA_OPTS": "-javaagent:${TRACER}"
19+
}
20+
},
21+
"profiling": {
22+
"env": {
23+
"VARIANT": "profiling",
24+
"JAVA_OPTS": "-javaagent:${TRACER} -Ddd.profiling.enabled=true"
25+
}
26+
},
27+
"appsec": {
28+
"env": {
29+
"VARIANT": "appsec",
30+
"JAVA_OPTS": "-javaagent:${TRACER} -Ddd.appsec.enabled=true"
31+
}
32+
},
33+
"iast": {
34+
"env": {
35+
"VARIANT": "iast",
36+
"JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true"
37+
}
38+
},
39+
"iast_FULL": {
40+
"env": {
41+
"VARIANT": "iast_FULL",
42+
"JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL"
43+
}
44+
},
45+
"iast_INACTIVE": {
46+
"env": {
47+
"VARIANT": "iast_INACTIVE",
48+
"JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=inactive"
49+
}
50+
}
51+
}
52+
}

‎benchmark/load/insecure-bank/k6.js‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import http from 'k6/http';
2+
import {checkResponse, isOk, isRedirect} from "../../utils/k6.js";
3+
4+
const baseUrl = 'http://localhost:8080';
5+
6+
export const options = {
7+
discardResponseBodies: true,
8+
vus: 5,
9+
iterations: 90000
10+
};
11+
12+
export default function () {
13+
14+
// login form
15+
const loginResponse = http.post(`${baseUrl}/login`, {
16+
username: 'john',
17+
password: 'test'
18+
}, {
19+
redirects: 0
20+
});
21+
checkResponse(loginResponse, isRedirect);
22+
23+
// dashboard
24+
const dashboard = http.get(`${baseUrl}/dashboard`);
25+
checkResponse(dashboard, isOk);
26+
27+
// logout
28+
const logout = http.get(`${baseUrl}/j_spring_security_logout`, {
29+
redirects: 0
30+
});
31+
checkResponse(logout, isRedirect);
32+
}

‎benchmark/load/petclinic/benchmark.json‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
"name": "load_petclinic",
3+
"setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"",
4+
"service": "bash -c \"${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"",
5+
"run": "bash -c \"java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log\"",
6+
"timeout": 150,
7+
"iterations": 1,
8+
"variants": {
9+
"${NO_AGENT_VARIANT}": {
10+
"env": {
11+
"VARIANT": "${NO_AGENT_VARIANT}",
12+
"JAVA_OPTS": ""
13+
}
14+
},
15+
"tracing": {
16+
"env": {
17+
"VARIANT": "tracing",
18+
"JAVA_OPTS": "-javaagent:${TRACER}"
19+
}
20+
},
21+
"profiling": {
22+
"env": {
23+
"VARIANT": "profiling",
24+
"JAVA_OPTS": "-javaagent:${TRACER} -Ddd.profiling.enabled=true"
25+
}
26+
},
27+
"appsec": {
28+
"env": {
29+
"VARIANT": "appsec",
30+
"JAVA_OPTS": "-javaagent:${TRACER} -Ddd.appsec.enabled=true"
31+
}
32+
},
33+
"iast": {
34+
"env": {
35+
"VARIANT": "iast",
36+
"JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true"
37+
}
38+
},
39+
"iast_FULL": {
40+
"env": {
41+
"VARIANT": "iast_FULL",
42+
"JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL"
43+
}
44+
},
45+
"iast_INACTIVE": {
46+
"env": {
47+
"VARIANT": "iast_INACTIVE",
48+
"JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=inactive"
49+
}
50+
}
51+
}
52+
}

‎benchmark/load/petclinic/k6.js‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import http from 'k6/http';
2+
import {checkResponse, isOk} from "../../utils/k6.js";
3+
4+
const baseUrl = 'http://localhost:8080';
5+
6+
export const options = {
7+
discardResponseBodies: true,
8+
vus: 5,
9+
iterations: 80000
10+
};
11+
12+
export default function () {
13+
14+
// find owner
15+
const ownersList = http.get(`${baseUrl}/owners?lastName=`);
16+
checkResponse(ownersList, isOk);
17+
}

‎benchmark/load/run.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/usr/bin/env bash
2+
set -eu
3+
4+
source "${UTILS_DIR}/update-java-version.sh" 17
5+
"${UTILS_DIR}/run-sirun-benchmarks.sh" "$@"

0 commit comments

Comments
 (0)