Skip to content

ROX-33655: Query changes for enhanced filters #19909

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ksurabhi91 wants to merge 1 commit into from
Closed

ROX-33655: Query changes for enhanced filters #19909

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 124 additions & 5 deletions central/reports/common/query_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"strings"
"time"

"github.com/pkg/errors"
collectionDataStore "github.com/stackrox/rox/central/resourcecollection/datastore"
v1 "github.com/stackrox/rox/generated/api/v1"
"github.com/stackrox/rox/generated/storage"
Expand All @@ -27,17 +28,20 @@ type queryBuilder struct {
collection *storage.ResourceCollection
collectionQueryResolver collectionDataStore.QueryResolver
dataStartTime time.Time
entityScope *storage.EntityScope
}

// NewVulnReportQueryBuilder builds a query builder to build scope and cve filtering queries for vuln reporting
func NewVulnReportQueryBuilder(collection *storage.ResourceCollection, vulnFilters *storage.VulnerabilityReportFilters,
func NewVulnReportQueryBuilder(collection *storage.ResourceCollection, entityScope *storage.EntityScope, vulnFilters *storage.VulnerabilityReportFilters,
collectionQueryRes collectionDataStore.QueryResolver, dataStartTime time.Time) *queryBuilder {
return &queryBuilder{
vulnFilters: vulnFilters,
collection: collection,
entityScope: entityScope,
vulnFilters: vulnFilters,
collectionQueryResolver: collectionQueryRes,
dataStartTime: dataStartTime,
}

}

// BuildQuery builds scope and cve filtering queries for vuln reporting
Expand All @@ -46,7 +50,13 @@ func (q *queryBuilder) BuildQuery(
clusters []effectiveaccessscope.Cluster,
namespaces []effectiveaccessscope.Namespace,
) (*ReportQuery, error) {
deploymentsQuery, err := q.collectionQueryResolver.ResolveCollectionQuery(ctx, q.collection)
deploymentsQuery := search.EmptyQuery()
var err error
if q.collection != nil {
deploymentsQuery, err = q.collectionQueryResolver.ResolveCollectionQuery(ctx, q.collection)
} else if q.entityScope != nil {
deploymentsQuery, err = q.buildEntityScopeQuery()
}
if err != nil {
return nil, err
}
Expand All @@ -66,7 +76,9 @@ func (q *queryBuilder) BuildQuery(
}, nil
}

func (q *queryBuilder) buildCVEAttributesQuery() (string, error) {
// addSeverityFixabilityFiltersCollectionScopedReports adds severity, fixability filters for collection scoped reports
func (q *queryBuilder) addSeverityFixabilityFiltersCollectionScopedReports() []string {

vulnReportFilters := q.vulnFilters
var conjuncts []string

Expand All @@ -86,13 +98,26 @@ func (q *queryBuilder) buildCVEAttributesQuery() (string, error) {
if len(severities) > 0 {
conjuncts = append(conjuncts, search.NewQueryBuilder().AddExactMatches(search.Severity, severities...).Query())
}
return conjuncts
}

func (q *queryBuilder) buildCVEAttributesQuery() (string, error) {

vulnReportFilters := q.vulnFilters
var conjuncts []string

if q.collection != nil {
// for collections only add fixability, severity filters for CVE
conjuncts = q.addSeverityFixabilityFiltersCollectionScopedReports()
} else if q.entityScope != nil {
// for entity scoped reports add all the search filters from query string
conjuncts = append(conjuncts, q.vulnFilters.GetQuery())
}
if filterVulnsByFirstOccurrenceTime(vulnReportFilters) {
startTimeStr := fmt.Sprintf(">=%s", q.dataStartTime.Format("01/02/2006 3:04:05 PM MST"))
tsQ := search.NewQueryBuilder().AddStrings(search.FirstImageOccurrenceTimestamp, startTimeStr).Query()
conjuncts = append(conjuncts, tsQ)
}

return strings.Join(conjuncts, "+"), nil
}

Expand Down Expand Up @@ -129,6 +154,100 @@ func (q *queryBuilder) buildAccessScopeQuery(
return scopeQuery, nil
}

// buildEntityScopeQuery uses entity scope object to build v1 query
func (q *queryBuilder) buildEntityScopeQuery() (*v1.Query, error) {
rules := q.entityScope.GetRules()
if len(rules) == 0 {
return search.EmptyQuery(), nil
}

var conjuncts []*v1.Query
for _, rule := range rules {
fieldLabel, err := entityScopeRuleToFieldLabel(rule)
if err != nil {
return nil, err
}
isMapField := fieldLabel == search.DeploymentLabel ||
fieldLabel == search.NamespaceLabel ||
fieldLabel == search.ClusterLabel ||
fieldLabel == search.DeploymentAnnotation ||
fieldLabel == search.NamespaceAnnotation

values := make([]string, 0, len(rule.GetValues()))
for _, rv := range rule.GetValues() {
val := rv.GetValue()
if rv.GetMatchType() == storage.MatchType_REGEX {
val = search.RegexPrefix + val
}
values = append(values, val)
}

if len(values) == 0 {
continue
}

if isMapField {
for _, v := range values {
key, value := splitLabelValue(v)
conjuncts = append(conjuncts,
search.NewQueryBuilder().AddMapQuery(fieldLabel, key, value).ProtoQuery())
}
} else if rule.GetValues()[0].GetMatchType() == storage.MatchType_REGEX {
conjuncts = append(conjuncts,
search.NewQueryBuilder().AddStrings(fieldLabel, values...).ProtoQuery())
} else {
conjuncts = append(conjuncts,
search.NewQueryBuilder().AddExactMatches(fieldLabel, values...).ProtoQuery())
}
}

if len(conjuncts) == 0 {
return search.EmptyQuery(), nil
}
return search.ConjunctionQuery(conjuncts...), nil
}

// entityScopeRuleToFieldLabel returns search filter for given entity field pair
func entityScopeRuleToFieldLabel(rule *storage.EntityScopeRule) (search.FieldLabel, error) {
switch rule.GetEntity() {
case storage.EntityType_ENTITY_TYPE_DEPLOYMENT:
switch rule.GetField() {
case storage.EntityField_FIELD_NAME:
return search.DeploymentName, nil
case storage.EntityField_FIELD_LABEL:
return search.DeploymentLabel, nil
case storage.EntityField_FIELD_ANNOTATION:
return search.DeploymentAnnotation, nil
}
case storage.EntityType_ENTITY_TYPE_NAMESPACE:
switch rule.GetField() {
case storage.EntityField_FIELD_NAME:
return search.Namespace, nil
case storage.EntityField_FIELD_LABEL:
return search.NamespaceLabel, nil
case storage.EntityField_FIELD_ANNOTATION:
return search.NamespaceAnnotation, nil
}
case storage.EntityType_ENTITY_TYPE_CLUSTER:
switch rule.GetField() {
case storage.EntityField_FIELD_NAME:
return search.Cluster, nil
case storage.EntityField_FIELD_LABEL:
return search.ClusterLabel, nil
}
}
return "", errors.Errorf("Unsupported entity/field combination %s/%s", rule.GetEntity(), rule.GetField())
}

func filterVulnsByFirstOccurrenceTime(vulnReportFilters *storage.VulnerabilityReportFilters) bool {
return vulnReportFilters.GetSinceLastSentScheduledReport() || vulnReportFilters.GetSinceStartDate() != nil
}

// split map field values like namespace labels(key=val) to key,val pair
func splitLabelValue(labelVal string) (string, string) {
parts := strings.SplitN(labelVal, "=", 2)
if len(parts) == 2 {
return fmt.Sprintf("%q", parts[0]), fmt.Sprintf("%q", parts[1])
}
return fmt.Sprintf("%q", labelVal), fmt.Sprintf("%q", "")
}
186 changes: 186 additions & 0 deletions central/reports/common/query_builder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,3 +197,189 @@ func TestBuildAccessScopeQuery(t *testing.T) {
func assertByDirectComparison(t testing.TB, expected *v1.Query, actual *v1.Query) {
protoassert.Equal(t, expected, actual)
}

func TestBuildEntityScopeQuery(t *testing.T) {
testCases := []struct {
name string
scope *storage.EntityScope
expected *v1.Query
assertQueries func(t testing.TB, expected *v1.Query, actual *v1.Query)
hasError bool
}{
{
name: "Empty rules returns empty query (match all)",
scope: &storage.EntityScope{},
expected: search.EmptyQuery(),
assertQueries: assertByDirectComparison,
},
{
name: "Namespace rule",
scope: &storage.EntityScope{
Rules: []*storage.EntityScopeRule{
{
Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE,
Field: storage.EntityField_FIELD_NAME,
Values: []*storage.RuleValue{
{Value: "prod", MatchType: storage.MatchType_EXACT},
{Value: "staging", MatchType: storage.MatchType_EXACT},
},
},
},
},
expected: search.NewQueryBuilder().AddExactMatches(search.Namespace, "prod", "staging").ProtoQuery(),
assertQueries: assertByDirectComparison,
},
{
name: "Single deployment name rule",
scope: &storage.EntityScope{
Rules: []*storage.EntityScopeRule{
{
Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT,
Field: storage.EntityField_FIELD_NAME,
Values: []*storage.RuleValue{
{Value: "web-server", MatchType: storage.MatchType_EXACT},
},
},
},
},
expected: search.NewQueryBuilder().AddExactMatches(search.DeploymentName, "web-server").ProtoQuery(),
assertQueries: assertByDirectComparison,
},
{
name: "Cluster name rule",
scope: &storage.EntityScope{
Rules: []*storage.EntityScopeRule{
{
Entity: storage.EntityType_ENTITY_TYPE_CLUSTER,
Field: storage.EntityField_FIELD_NAME,
Values: []*storage.RuleValue{
{Value: "prod-us", MatchType: storage.MatchType_EXACT},
{Value: "prod-eu", MatchType: storage.MatchType_EXACT},
},
},
},
},
expected: search.NewQueryBuilder().AddExactMatches(search.Cluster, "prod-us", "prod-eu").ProtoQuery(),
assertQueries: assertByDirectComparison,
},
{
name: "Multiple rules are ANDed",
scope: &storage.EntityScope{
Rules: []*storage.EntityScopeRule{
{
Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE,
Field: storage.EntityField_FIELD_NAME,
Values: []*storage.RuleValue{
{Value: "prod", MatchType: storage.MatchType_EXACT},
},
},
{
Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT,
Field: storage.EntityField_FIELD_NAME,
Values: []*storage.RuleValue{
{Value: "backend", MatchType: storage.MatchType_EXACT},
{Value: "frontend", MatchType: storage.MatchType_EXACT},
},
},
},
},
expected: search.ConjunctionQuery(
search.NewQueryBuilder().AddExactMatches(search.Namespace, "prod").ProtoQuery(),
search.NewQueryBuilder().AddExactMatches(search.DeploymentName, "backend", "frontend").ProtoQuery(),
),
assertQueries: assertByDirectComparison,
},
{
name: "Label rule uses map query",
scope: &storage.EntityScope{
Rules: []*storage.EntityScopeRule{
{
Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE,
Field: storage.EntityField_FIELD_LABEL,
Values: []*storage.RuleValue{
{Value: "env=prod", MatchType: storage.MatchType_EXACT},
},
},
},
},
expected: search.NewQueryBuilder().AddMapQuery(search.NamespaceLabel, `"env"`, `"prod"`).ProtoQuery(),
assertQueries: assertByDirectComparison,
},
{
name: "Regex match type adds r/ prefix",
scope: &storage.EntityScope{
Rules: []*storage.EntityScopeRule{
{
Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT,
Field: storage.EntityField_FIELD_NAME,
Values: []*storage.RuleValue{
{Value: "web-.*", MatchType: storage.MatchType_REGEX},
},
},
},
},
expected: search.NewQueryBuilder().AddStrings(search.DeploymentName, "r/web-.*").ProtoQuery(),
assertQueries: assertByDirectComparison,
},
{
name: "Rule with empty values is skipped",
scope: &storage.EntityScope{
Rules: []*storage.EntityScopeRule{
{
Entity: storage.EntityType_ENTITY_TYPE_NAMESPACE,
Field: storage.EntityField_FIELD_NAME,
Values: []*storage.RuleValue{},
},
},
},
expected: search.EmptyQuery(),
assertQueries: assertByDirectComparison,
},
{
name: "Unsupported entity/field returns error",
scope: &storage.EntityScope{
Rules: []*storage.EntityScopeRule{
{
Entity: storage.EntityType_ENTITY_TYPE_CLUSTER,
Field: storage.EntityField_FIELD_ANNOTATION,
Values: []*storage.RuleValue{
{Value: "team=infra", MatchType: storage.MatchType_EXACT},
},
},
},
},
hasError: true,
},
{
name: "Deployment annotation rule",
scope: &storage.EntityScope{
Rules: []*storage.EntityScopeRule{
{
Entity: storage.EntityType_ENTITY_TYPE_DEPLOYMENT,
Field: storage.EntityField_FIELD_ANNOTATION,
Values: []*storage.RuleValue{
{Value: "owner=team-a", MatchType: storage.MatchType_EXACT},
},
},
},
},
expected: search.NewQueryBuilder().AddMapQuery(search.DeploymentAnnotation, `"owner"`, `"team-a"`).ProtoQuery(),
assertQueries: assertByDirectComparison,
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
qb := &queryBuilder{
entityScope: tc.scope,
}
result, err := qb.buildEntityScopeQuery()
if tc.hasError {
assert.Error(t, err)
return
}
assert.NoError(t, err)
tc.assertQueries(t, tc.expected, result)
})
}
}
2 changes: 1 addition & 1 deletion central/reports/scheduler/schedule.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,7 +361,7 @@ func (s *scheduler) getReportData(ctx context.Context, rc *storage.ReportConfigu

func (s *scheduler) buildReportQuery(ctx context.Context, rc *storage.ReportConfiguration,
collection *storage.ResourceCollection) (*common.ReportQuery, error) {
qb := common.NewVulnReportQueryBuilder(collection, rc.GetVulnReportFilters(), s.collectionQueryResolver,
qb := common.NewVulnReportQueryBuilder(collection, nil, rc.GetVulnReportFilters(), s.collectionQueryResolver,
timestamp.FromProtobuf(rc.GetLastSuccessfulRunTime()).GoTime())
rQuery, err := qb.BuildQuery(ctx, nil, nil)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion central/reports/scheduler/v2/reportgenerator/report_gen_impl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ func (rg *reportGeneratorImpl) buildReportQueryViewBased(snap *storage.ReportSna

func (rg *reportGeneratorImpl) buildReportQuery(snap *storage.ReportSnapshot,
collection *storage.ResourceCollection, dataStartTime time.Time) (*common.ReportQuery, error) {
qb := common.NewVulnReportQueryBuilder(collection, snap.GetVulnReportFilters(), rg.collectionQueryResolver,
qb := common.NewVulnReportQueryBuilder(collection, snap.GetResourceScope().GetEntityScope(), snap.GetVulnReportFilters(), rg.collectionQueryResolver,
dataStartTime)
allClusters, allNamespaces, err := rg.getClustersAndNamespacesForSAC()
if err != nil {
Expand Down
Loading
Loading