Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .openshift-ci/begin.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,11 @@ fi
if [[ "${JOB_NAME:-}" =~ -ocp- ]]; then
info "Setting worker node type and count for OCP 4 jobs"
set_ci_shared_export WORKER_NODE_COUNT 2
set_ci_shared_export WORKER_NODE_TYPE e2-standard-8
if [[ "${JOB_NAME:-}" =~ vm-scanning ]]; then
set_ci_shared_export WORKER_NODE_TYPE n2-standard-8
else
set_ci_shared_export WORKER_NODE_TYPE e2-standard-8
fi
fi

if [[ "${JOB_NAME:-}" =~ -eks- ]]; then
Expand Down
14 changes: 14 additions & 0 deletions .openshift-ci/ci_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,20 @@ def run(self):
)


class VMScanningE2e(BaseTest):
TEST_TIMEOUT = 2 * 60 * 60
TEST_OUTPUT_DIR = "/tmp/vm-scanning-test-logs"

def run(self):
print("Executing VM scanning e2e tests")

self.run_with_graceful_kill(
["tests/e2e/run-vm-scanning.sh", self.TEST_OUTPUT_DIR],
self.TEST_TIMEOUT,
output_dir=self.TEST_OUTPUT_DIR,
)


class SensorIntegration(BaseTest):
TEST_TIMEOUT = 90 * 60
TEST_OUTPUT_DIR = "/tmp/sensor-integration-test-logs"
Expand Down
6 changes: 4 additions & 2 deletions .openshift-ci/dispatch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ ci_export CI_JOB_NAME "$ci_job"

case "$ci_job" in
gke*qa-e2e-tests|gke*nongroovy-e2e-tests|gke*upgrade-tests|gke-ui-e2e-tests|\
eks-qa-e2e-tests|osd*qa-e2e-tests|gke*sensor-integration-tests)
eks-qa-e2e-tests|osd*qa-e2e-tests|gke*sensor-integration-tests|\
ocp*vm-scanning-e2e-tests)
openshift_ci_e2e_mods
;;
*-operator-e2e-tests)
Expand All @@ -40,7 +41,8 @@ case "$ci_job" in
esac

case "$ci_job" in
eks-qa-e2e-tests|osd*qa-e2e-tests|ocp*ui-e2e-tests)
eks-qa-e2e-tests|osd*qa-e2e-tests|ocp*ui-e2e-tests|\
ocp*vm-scanning-e2e-tests)
setup_automation_flavor_e2e_cluster "$ci_job"
;;
esac
Expand Down
83 changes: 83 additions & 0 deletions scripts/ci/jobs/ocp_vm_scanning_e2e_tests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#!/usr/bin/env -S python3 -u

"""
Run VM scanning E2E tests on an OpenShift cluster with CNV / KubeVirt and VSOCK enabled.

The CNV operator is installed automatically when INSTALL_CNV_OPERATOR=true (set below).
If already present on the cluster, the existing installation is reused.
VSOCK prerequisites are verified at Go test startup (mustVerifyClusterVSOCKReady) and the
suite fails fast with actionable diagnostics if they are not met.
"""
import os

from runners import ClusterTestRunner
from clusters import AutomationFlavorsCluster
from pre_tests import PreSystemTests
from ci_tests import VMScanningE2e
from post_tests import PostClusterTest, FinalPost

os.environ["DEPLOY_STACKROX_VIA_OPERATOR"] = "true"
os.environ["ORCHESTRATOR_FLAVOR"] = "openshift"
os.environ["SENSOR_SCANNER_SUPPORT"] = "true"
os.environ["ROX_DEPLOY_SENSOR_WITH_CRS"] = "true"
os.environ["SENSOR_HELM_MANAGED"] = "true"
os.environ["INSTALL_CNV_OPERATOR"] = "true"
os.environ["ROX_VIRTUAL_MACHINES"] = "true"
# TODO: Move images to quay.io/rhacs-eng/ before merging to main.
os.environ["VM_IMAGE_RHEL9"] = "quay.io/prygiels/rhel9-dnf-primed:latest"
os.environ["VM_IMAGE_RHEL10"] = "quay.io/prygiels/rhel10-dnf-primed:latest"


class VMScanningPostTest(PostClusterTest):
"""Extends standard post-test with CNV namespace logs and VM artifacts."""

VM_ARTIFACTS_DIR = "/tmp/vm-scanning-artifacts"

def __init__(self, **kwargs):
super().__init__(**kwargs)
self.k8s_namespaces.extend([
"openshift-cnv",
])

def run(self, test_outputs=None):
self.collect_vm_artifacts()
super().run(test_outputs=test_outputs)

def collect_vm_artifacts(self):
"""Collect VM/VMI descriptions and events from test namespaces."""
artifacts_dir = self.VM_ARTIFACTS_DIR
os.makedirs(artifacts_dir, exist_ok=True)
for resource in [
"vm", "vmi", "datavolume",
]:
self.run_with_best_effort(
[
"bash", "-c",
f"kubectl get {resource} --all-namespaces -o wide "
f"> {artifacts_dir}/{resource}-list.txt 2>&1 || true; "
f"kubectl describe {resource} --all-namespaces "
f"> {artifacts_dir}/{resource}-describe.txt 2>&1 || true",
],
timeout=self.COLLECT_TIMEOUT,
)
self.run_with_best_effort(
[
"bash", "-c",
f"kubectl get events --all-namespaces "
f"--field-selector involvedObject.kind=VirtualMachineInstance "
f"> {artifacts_dir}/vmi-events.txt 2>&1 || true",
],
timeout=self.COLLECT_TIMEOUT,
)
self.data_to_store.append(artifacts_dir)


ClusterTestRunner(
cluster=AutomationFlavorsCluster(),
pre_test=PreSystemTests(),
test=VMScanningE2e(),
post_test=VMScanningPostTest(
check_stackrox_logs=False,
),
final_post=FinalPost(),
).run()
89 changes: 89 additions & 0 deletions test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
#!/usr/bin/env bash

set -euo pipefail

# Required to start:
# - Create a new OCP cluster
# - install virtualization operator
# - set KUBECONFIG
# - Deploy ACS
# - Enable ROX_VIRTUAL_MACHINES

# Deployment behavior (matches CI lane intent)
export ORCHESTRATOR_FLAVOR=openshift
export DEPLOY_STACKROX_VIA_OPERATOR=true
export SENSOR_SCANNER_SUPPORT=true
export ROX_DEPLOY_SENSOR_WITH_CRS=true
export SENSOR_HELM_MANAGED=true
export ROX_VIRTUAL_MACHINES=true

# VM-scanning required inputs
export VIRTCTL_PATH="$(command -v virtctl)"
export ROXAGENT_BINARY_PATH="$PWD/bin/linux_amd64/roxagent"

# From your cluster's virtualization boot sources:
# export VM_IMAGE_RHEL9="$(oc get istag -n openshift-virtualization-os-images rhel9-guest:latest -o jsonpath='{.image.dockerImageReference}')"
export VM_IMAGE_RHEL9="quay.io/prygiels/rhel9-dnf-primed:latest"
# export VM_IMAGE_RHEL10="$(oc get istag -n openshift-virtualization-os-images rhel10-guest:latest -o jsonpath='{.image.dockerImageReference}')"
export VM_IMAGE_RHEL10="quay.io/prygiels/rhel10-dnf-primed:latest"
export VM_IMAGE_PULL_SECRET_PATH="$HOME/.config/containers/auth.json"

# Guest users for RHEL cloud images:
export VM_GUEST_USER_RHEL9="cloud-user"
export VM_GUEST_USER_RHEL10="cloud-user"

export API_ENDPOINT="$(oc -n stackrox get route central -o jsonpath='{.spec.host}'):443"
export ROX_USERNAME="admin"
export ROX_ADMIN_PASSWORD="admin"

# Persistent, dedicated SSH keypair for VM scanning E2E.
# Reused across runs so manual troubleshooting can reuse the same key path.
vm_scan_ssh_key="$HOME/.ssh/id_stackrox_vm_scan_e2e_rsa"
mkdir -p "$HOME/.ssh"
chmod 700 "$HOME/.ssh"
if [[ ! -f "$vm_scan_ssh_key" ]]; then
ssh-keygen -q -t rsa -b 4096 -N "" -f "$vm_scan_ssh_key" -C "stackrox-vm-scan-e2e" >/dev/null
fi
if [[ ! -f "${vm_scan_ssh_key}.pub" ]]; then
ssh-keygen -y -f "$vm_scan_ssh_key" > "${vm_scan_ssh_key}.pub"
fi
chmod 600 "$vm_scan_ssh_key"
chmod 644 "${vm_scan_ssh_key}.pub"

export VM_SSH_PRIVATE_KEY="$(cat "$vm_scan_ssh_key")" # PEM content of the private key
export VM_SSH_PUBLIC_KEY="$(cat "${vm_scan_ssh_key}.pub")" # authorized_keys line for cloud-init


export ROXAGENT_REPO2CPE_PRIMARY_URL="https://security.access.redhat.com/data/metrics/repository-to-cpe.json"
export ROXAGENT_REPO2CPE_FALLBACK_URL="https://security.access.redhat.com/data/metrics/repository-to-cpe.json"
export ROXAGENT_REPO2CPE_PRIMARY_ATTEMPTS=3

export VM_SCAN_NAMESPACE_PREFIX="vm-scan-e2e"
# TEMPORARY local-dev override: force one namespace across manual runs.
export VM_SCAN_NAMESPACE="vm-scan-e2e-manual"
export VM_SCAN_TIMEOUT=20m
export VM_SCAN_POLL_INTERVAL=10s
export VM_SCAN_ESCALATION_ATTEMPT=5
export VM_DELETE_TIMEOUT=5m
export VM_SCAN_SKIP_CLEANUP=true # keep VMs and namespace after test run for faster iteration

echo "Manual SSH (rhel9):"
echo "${VIRTCTL_PATH} ssh -n ${VM_SCAN_NAMESPACE} --identity-file \"${vm_scan_ssh_key}\" --username \"${VM_GUEST_USER_RHEL9}\" vmi/vm-rhel9"
echo
echo "Manual SSH command check (sudo):"
echo "${VIRTCTL_PATH} ssh -n ${VM_SCAN_NAMESPACE} --identity-file \"${vm_scan_ssh_key}\" --username \"${VM_GUEST_USER_RHEL9}\" vmi/vm-rhel9 --command '\"sudo\" \"-n\" \"true\"'"
echo
echo "Building roxagent..."
make roxagent_linux-amd64
echo
echo "Running vmhelpers tests..."
go test -race -p 1 -timeout 90m ./tests/vmhelpers -v
echo
echo "Running unit tests..."
go test -tags test -race -count=1 -v ./tests/testmetrics
go test -tags test_e2e ./tests -run TestLoadVMScanConfig -v
echo
echo "Running vmscanning tests..."
go test -tags test_e2e -run TestVMScanning -v -count=1 -p 1 -timeout 120m ./tests

echo "Done!"
12 changes: 12 additions & 0 deletions tests/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,16 @@ compliance-v2-tests:
@GOTAGS=$(GOTAGS),test,compliance ../scripts/go-test.sh -timeout 30m -cover -v -run TestComplianceV2 2>&1 | tee test.log
@$(MAKE) report JUNIT_OUT=compliance-v2-tests-results

.PHONY: vm-scanning-unit-tests
vm-scanning-unit-tests:
@echo "+ $@"
@GOTAGS=$(GOTAGS),test $(TOPLEVEL)/scripts/go-test.sh -cover $(TESTFLAGS) -v ./vmhelpers ./testmetrics 2>&1 | tee test.log
@$(MAKE) report JUNIT_OUT=vm-scanning-unit-tests-results

.PHONY: vm-scanning-tests
vm-scanning-tests:
@echo "+ $@"
@GOTAGS=$(GOTAGS),test,test_e2e $(TOPLEVEL)/scripts/go-test.sh -cover $(TESTFLAGS) -v -run 'TestVMScanning|TestLoadVMScanConfig' $(shell go list -e ./... | grep -v generated | grep -v vendor | grep -v vmhelpers) 2>&1 | tee test.log
@$(MAKE) report JUNIT_OUT=vm-scanning-tests-results

include ../make/stackrox.mk
71 changes: 71 additions & 0 deletions tests/e2e/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,10 @@ deploy_central_via_operator() {
customize_envVars+=$'\n value: "true"'
customize_envVars+=$'\n - name: ROX_TAILORED_PROFILES'
customize_envVars+=$'\n value: "true"'
if [[ "${ROX_VIRTUAL_MACHINES:-}" == "true" ]]; then
customize_envVars+=$'\n - name: ROX_VIRTUAL_MACHINES'
customize_envVars+=$'\n value: "true"'
fi

local scannerV4ScannerComponent="Default"
case "${ROX_SCANNER_V4:-}" in
Expand Down Expand Up @@ -482,6 +486,10 @@ deploy_sensor_via_operator() {
fi

customize_envVars=""
if [[ "${ROX_VIRTUAL_MACHINES:-}" == "true" ]]; then
customize_envVars+=$'\n - name: ROX_VIRTUAL_MACHINES'
customize_envVars+=$'\n value: "true"'
fi
if [[ -n "${ROX_NETFLOW_BATCHING:-}" ]]; then
customize_envVars+=$'\n - name: ROX_NETFLOW_BATCHING'
customize_envVars+=$'\n value: "'"${ROX_NETFLOW_BATCHING}"'"'
Expand Down Expand Up @@ -568,6 +576,12 @@ deploy_optional_e2e_components() {
else
info "Skipping the compliance operator install"
fi

if [[ "${INSTALL_CNV_OPERATOR:-false}" == "true" ]]; then
install_the_cnv_operator
else
info "Skipping the CNV operator install"
fi
}

install_the_compliance_operator() {
Expand All @@ -590,6 +604,63 @@ install_the_compliance_operator() {
oc get csv -n openshift-compliance
}

install_the_cnv_operator() {
local csv
csv=$(oc get csv -n openshift-cnv -o json 2>/dev/null | jq -r '.items[] | select(.metadata.name | test("kubevirt-hyperconverged")).metadata.name // empty')
if [[ -z "$csv" ]]; then
info "Installing the OpenShift Virtualization (CNV) operator"
oc apply -f "${ROOT}/tests/e2e/yaml/cnv-operator/namespace.yaml"
oc apply -f "${ROOT}/tests/e2e/yaml/cnv-operator/operator-group.yaml"
oc apply -f "${ROOT}/tests/e2e/yaml/cnv-operator/subscription.yaml"
info "Waiting for CNV operator deployment (this may take several minutes)..."
wait_for_object_to_appear openshift-cnv deploy/hco-operator 900
oc rollout status deploy/hco-operator -n openshift-cnv --timeout=300s
info "Creating HyperConverged CR..."
oc apply -f "${ROOT}/tests/e2e/yaml/cnv-operator/hyperconverged.yaml"
info "Waiting for virt-operator deployment..."
wait_for_object_to_appear openshift-cnv deploy/virt-operator 600
oc rollout status deploy/virt-operator -n openshift-cnv --timeout=300s
info "Waiting for virt-handler daemonset..."
wait_for_object_to_appear openshift-cnv ds/virt-handler 600
oc rollout status ds/virt-handler -n openshift-cnv --timeout=600s
else
info "Reusing existing CNV operator deployment from $csv subscription"
fi

# Ensure VSOCK feature gate via annotation on the HyperConverged CR.
# The HC operator reconciles the KubeVirt CR, so patching KubeVirt
# directly is ephemeral. The jsonpatch annotation is the supported way
# to inject custom feature gates into the managed KubeVirt CR.
local vsock_patch='[{"op":"add","path":"/spec/configuration/developerConfiguration/featureGates/-","value":"VSOCK"}]'
local kv_gates
kv_gates=$(oc get kubevirt -n openshift-cnv \
-o jsonpath='{.items[0].spec.configuration.developerConfiguration.featureGates}' 2>/dev/null || true)
if [[ "$kv_gates" != *"VSOCK"* ]]; then
info "Annotating HyperConverged CR to add VSOCK feature gate..."
oc annotate hyperconverged kubevirt-hyperconverged -n openshift-cnv --overwrite \
"kubevirt.kubevirt.io/jsonpatch=${vsock_patch}"
info "Waiting for VSOCK to appear in KubeVirt CR feature gates..."
local attempts=0
while (( attempts < 60 )); do
kv_gates=$(oc get kubevirt -n openshift-cnv \
-o jsonpath='{.items[0].spec.configuration.developerConfiguration.featureGates}' 2>/dev/null || true)
if [[ "$kv_gates" == *"VSOCK"* ]]; then
break
fi
(( attempts++ ))
sleep 5
done
if [[ "$kv_gates" != *"VSOCK"* ]]; then
die "KubeVirt CR still missing VSOCK after 5 minutes"
fi
info "Waiting for virt-handler rollout after VSOCK enablement..."
oc rollout status ds/virt-handler -n openshift-cnv --timeout=600s
else
info "KubeVirt CR already has VSOCK feature gate"
fi
oc get csv -n openshift-cnv
}

setup_client_CA_auth_provider() {
info "Set up client CA auth provider for endpoints_test.go"

Expand Down
47 changes: 47 additions & 0 deletions tests/e2e/run-vm-scanning.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/usr/bin/env bash
# shellcheck disable=SC1091

set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)"
# shellcheck source=../../scripts/lib.sh
source "$ROOT/scripts/lib.sh"
# shellcheck source=../../scripts/ci/sensor-wait.sh
source "$ROOT/scripts/ci/sensor-wait.sh"
# shellcheck source=../../tests/scripts/setup-certs.sh
source "$ROOT/tests/scripts/setup-certs.sh"
# shellcheck source=../../tests/e2e/lib.sh
source "$ROOT/tests/e2e/lib.sh"
# shellcheck source=../../tests/e2e/vm-scanning-lib.sh
source "$ROOT/tests/e2e/vm-scanning-lib.sh"

test_vm_scanning_e2e() {
local output_dir="${1:-vm-scanning-tests-results}"

info "Starting VM scanning e2e tests"

export_test_environment
setup_deployment_env true false
ensure_vm_scanning_cluster_prereqs
remove_existing_stackrox_resources
setup_default_TLS_certs

deploy_optional_e2e_components

verify_kvm_available
ensure_virtctl

deploy_stackrox

ensure_roxagent

cd "$ROOT"
rm -f FAIL
make -C tests TESTFLAGS="-race -p 1 -timeout 90m" vm-scanning-unit-tests || touch FAIL
store_test_results "tests/vm-scanning-unit-tests-results" "vm-scanning-unit-tests-results"
make -C tests TESTFLAGS="-race -p 1 -timeout 90m" vm-scanning-tests || touch FAIL
store_test_results "tests/vm-scanning-tests-results" "$output_dir"
[[ ! -f FAIL ]] || die "VM scanning e2e tests failed"
}

test_vm_scanning_e2e "${1:-vm-scanning-tests-results}"
Loading
Loading