Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Only add necessary attributes in DataFilterExtension #8769

Merged
merged 4 commits into from Apr 9, 2024
Merged

Only add necessary attributes in DataFilterExtension #8769

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
84c88fb
Only add necessary attributes in DataFilterExtension
felixpalmer Apr 8, 2024
5f823c8
Type defines
felixpalmer Apr 8, 2024
7f68d08
More render tests
felixpalmer Apr 8, 2024
9a34aec
Merge branch 'master' into felix/data-filter-opts
felixpalmer Apr 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/api-reference/extensions/data-filter-extension.md
View file
Open in desktop
Expand Up @@ -70,8 +70,8 @@ new deck.DataFilterExtension({});
new DataFilterExtension({filterSize, fp64});
```

* `filterSize` (number) - the size of the filter (number of columns to filter by). The data filter can show/hide data based on 1-4 numeric properties of each object. Default `1`.
* `categorySize` (number) - the size of the category filter (number of columns to filter by). The category filter can show/hide data based on 1-4 properties of each object. Default `1`.
* `filterSize` (number) - the size of the filter (number of columns to filter by). The data filter can show/hide data based on 1-4 numeric properties of each object. Set to `0` to disable numeric filtering. Default `1`.
* `categorySize` (number) - the size of the category filter (number of columns to filter by). The category filter can show/hide data based on 1-4 properties of each object. Set to `0` to disable category filtering. Default `0`.
* `fp64` (boolean) - if `true`, use 64-bit precision instead of 32-bit. Default `false`. See the "remarks" section below for use cases and limitations.
* `countItems` (boolean) - if `true`, reports the number of filtered objects with the `onFilteredItemsChange` callback. Default `false`.

Expand Down
106 changes: 57 additions & 49 deletions modules/extensions/src/data-filter/data-filter-extension.ts
View file
Open in desktop
Expand Up @@ -22,7 +22,7 @@ import type {Framebuffer} from '@luma.gl/core';
import type {Model} from '@luma.gl/engine';
import type {Layer, LayerContext, Accessor, UpdateParameters} from '@deck.gl/core';
import {_deepEqual as deepEqual, LayerExtension, log} from '@deck.gl/core';
import {shaderModule, shaderModule64} from './shader-module';
import {Defines, shaderModule, shaderModule64} from './shader-module';
import * as aggregator from './aggregator';

const defaultProps = {
Expand Down Expand Up @@ -96,15 +96,15 @@ export type DataFilterExtensionProps<DataT = any> = {

type DataFilterExtensionOptions = {
/**
* The size of the category filter (number of columns to filter by). The category filter can show/hide data based on 1-4 properties of each object.
* @default 1
* The size of the category filter (number of columns to filter by). The category filter can show/hide data based on 1-4 properties of each object. Set to `0` to disable category filtering.
* @default 0
*/
categorySize?: 1 | 2 | 3 | 4;
categorySize?: 0 | 1 | 2 | 3 | 4;
/**
* The size of the filter (number of columns to filter by). The data filter can show/hide data based on 1-4 numeric properties of each object.
* The size of the filter (number of columns to filter by). The data filter can show/hide data based on 1-4 numeric properties of each object. Set to `0` to disable numeric filtering.
* @default 1
*/
filterSize?: 1 | 2 | 3 | 4;
filterSize?: 0 | 1 | 2 | 3 | 4;
/**
* Use 64-bit precision instead of 32-bit.
* @default false
Expand All @@ -118,17 +118,17 @@ type DataFilterExtensionOptions = {
};

const defaultOptions: Required<DataFilterExtensionOptions> = {
categorySize: 1,
categorySize: 0,
filterSize: 1,
fp64: false,
countItems: false
};

const DATA_TYPE_FROM_SIZE = {
1: 'float',
2: 'vec2',
3: 'vec3',
4: 'vec4'
1: 'float' as const,
2: 'vec2' as const,
3: 'vec3' as const,
4: 'vec4' as const
};

/** Adds GPU-based data filtering functionalities to layers. It allows the layer to show/hide objects based on user-defined properties. */
Expand All @@ -144,54 +144,62 @@ export default class DataFilterExtension extends LayerExtension<

getShaders(this: Layer<DataFilterExtensionProps>, extension: this): any {
const {categorySize, filterSize, fp64} = extension.opts;
const defines: Defines = {};
if (categorySize) {
defines.DATACATEGORY_TYPE = DATA_TYPE_FROM_SIZE[categorySize];
defines.DATACATEGORY_CHANNELS = categorySize;
}
if (filterSize) {
defines.DATAFILTER_TYPE = DATA_TYPE_FROM_SIZE[filterSize];
defines.DATAFILTER_DOUBLE = Boolean(fp64);
}

return {
modules: [fp64 ? shaderModule64 : shaderModule],
defines: {
DATACATEGORY_TYPE: DATA_TYPE_FROM_SIZE[categorySize],
DATACATEGORY_CHANNELS: categorySize,
DATAFILTER_TYPE: DATA_TYPE_FROM_SIZE[filterSize],
DATAFILTER_DOUBLE: Boolean(fp64)
}
};
return {modules: [fp64 ? shaderModule64 : shaderModule], defines};
}

initializeState(this: Layer<DataFilterExtensionProps>, context: LayerContext, extension: this) {
const attributeManager = this.getAttributeManager();
const {categorySize, filterSize, fp64} = extension.opts;

if (attributeManager) {
attributeManager.add({
filterValues: {
size: filterSize,
type: fp64 ? 'float64' : 'float32',
accessor: 'getFilterValue',
shaderAttributes: {
filterValues: {
divisor: 0
},
instanceFilterValues: {
divisor: 1
if (filterSize) {
attributeManager.add({
filterValues: {
size: filterSize,
type: fp64 ? 'float64' : 'float32',
accessor: 'getFilterValue',
shaderAttributes: {
filterValues: {
divisor: 0
},
instanceFilterValues: {
divisor: 1
}
}
}
},
filterCategoryValues: {
size: categorySize,
accessor: 'getFilterCategory',
transform:
categorySize === 1
? d => extension._getCategoryKey.call(this, d, 0)
: d => d.map((x, i) => extension._getCategoryKey.call(this, x, i)),
shaderAttributes: {
filterCategoryValues: {
divisor: 0
},
instanceFilterCategoryValues: {
divisor: 1
});
}

if (categorySize) {
attributeManager.add({
filterCategoryValues: {
size: categorySize,
accessor: 'getFilterCategory',
transform:
categorySize === 1
? d => extension._getCategoryKey.call(this, d, 0)
: d => d.map((x, i) => extension._getCategoryKey.call(this, x, i)),
shaderAttributes: {
filterCategoryValues: {
divisor: 0
},
instanceFilterCategoryValues: {
divisor: 1
}
}
}
}
});
});
}
}

const {device} = this.context;
Expand Down Expand Up @@ -240,7 +248,7 @@ export default class DataFilterExtension extends LayerExtension<
if (this.state.filterModel) {
const filterNeedsUpdate =
// attributeManager must be defined for filterModel to be set
attributeManager!.attributes.filterValues.needsUpdate() ||
attributeManager!.attributes.filterValues?.needsUpdate() ||
attributeManager!.attributes.filterCategoryValues?.needsUpdate() ||
props.filterEnabled !== oldProps.filterEnabled ||
props.filterRange !== oldProps.filterRange ||
Expand Down Expand Up @@ -294,7 +302,7 @@ export default class DataFilterExtension extends LayerExtension<
filterModel.updateModuleSettings(params.moduleParameters);
// @ts-expect-error filterValue and filterIndices should always have buffer value
filterModel.setAttributes({
...filterValues.getValue(),
...filterValues?.getValue(),
...filterCategoryValues?.getValue(),
...filterIndices?.getValue()
});
Expand Down
139 changes: 98 additions & 41 deletions modules/extensions/src/data-filter/shader-module.ts
View file
Open in desktop
Expand Up @@ -5,34 +5,80 @@ import {glsl} from '../utils/syntax-tags';
/*
* data filter shader module
*/
export type Defines = {
// Defines passed externally
/**
* Primitive type of parameter used for category filtering. If undefined, category filtering disabled.
*/
DATACATEGORY_TYPE?: 'float' | 'vec2' | 'vec3' | 'vec4';
/**
* Number of category filtering channels. Must match dimension of `DATACATEGORY_TYPE`
*/
DATACATEGORY_CHANNELS?: 1 | 2 | 3 | 4;

/**
* Primitive type of parameter used for numeric filtering. If undefined, numeric filtering disabled.
*/
DATAFILTER_TYPE?: 'float' | 'vec2' | 'vec3' | 'vec4';

/**
* Enable 64-bit precision in numeric filter.
*/
DATAFILTER_DOUBLE?: boolean;

// Defines derived in shader
/**
* Numeric filter attribute
*/
DATAFILTER_ATTRIB?: 'filterValues' | 'instanceFilterValues';
/**
* Numeric filter attribute (low bits). Only used when `DATAFILTER_DOUBLE = true`
*/
DATAFILTER_ATTRIB_64LOW?: 'filterValues64Low' | 'instanceFilterValues64Low';
/**
* Category filter attribute
*/
DATACATEGORY_ATTRIB?: 'filterCategoryValues' | 'instanceFilterCategoryValues';
};

const vs = glsl`
uniform DATAFILTER_TYPE filter_min;
uniform DATAFILTER_TYPE filter_softMin;
uniform DATAFILTER_TYPE filter_softMax;
uniform DATAFILTER_TYPE filter_max;
uniform bool filter_useSoftMargin;
uniform bool filter_enabled;
uniform bool filter_transformSize;
uniform ivec4 filter_categoryBitMask;

#ifdef NON_INSTANCED_MODEL
#define DATAFILTER_ATTRIB filterValues
#define DATAFILTER_ATTRIB_64LOW filterValues64Low
#define DATACATEGORY_ATTRIB filterCategoryValues
#else
#define DATAFILTER_ATTRIB instanceFilterValues
#define DATAFILTER_ATTRIB_64LOW instanceFilterValues64Low
#define DATACATEGORY_ATTRIB instanceFilterCategoryValues
#ifdef DATAFILTER_TYPE
uniform DATAFILTER_TYPE filter_min;
uniform DATAFILTER_TYPE filter_softMin;
uniform DATAFILTER_TYPE filter_softMax;
uniform DATAFILTER_TYPE filter_max;

#ifdef NON_INSTANCED_MODEL
#define DATAFILTER_ATTRIB filterValues
#define DATAFILTER_ATTRIB_64LOW filterValues64Low
#else
#define DATAFILTER_ATTRIB instanceFilterValues
#define DATAFILTER_ATTRIB_64LOW instanceFilterValues64Low
#endif

in DATAFILTER_TYPE DATAFILTER_ATTRIB;
#ifdef DATAFILTER_DOUBLE
in DATAFILTER_TYPE DATAFILTER_ATTRIB_64LOW;

uniform DATAFILTER_TYPE filter_min64High;
felixpalmer marked this conversation as resolved.
Show resolved Hide resolved
uniform DATAFILTER_TYPE filter_max64High;
#endif
#endif

in DATAFILTER_TYPE DATAFILTER_ATTRIB;
#ifdef DATAFILTER_DOUBLE
in DATAFILTER_TYPE DATAFILTER_ATTRIB_64LOW;

uniform DATAFILTER_TYPE filter_min64High;
uniform DATAFILTER_TYPE filter_max64High;
#ifdef DATACATEGORY_TYPE
#ifdef NON_INSTANCED_MODEL
#define DATACATEGORY_ATTRIB filterCategoryValues
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking at this, I ask myself if we really want to require an attribute that can be applied in both instanced and non-instanced usage should be renamed. This simple renaming adds a lot of clutter to this shader.

#else
#define DATACATEGORY_ATTRIB instanceFilterCategoryValues
#endif
in DATACATEGORY_TYPE DATACATEGORY_ATTRIB;
#endif
in DATACATEGORY_TYPE DATACATEGORY_ATTRIB;

out float dataFilter_value;

Expand All @@ -48,45 +94,48 @@ float dataFilter_reduceValue(vec3 value) {
float dataFilter_reduceValue(vec4 value) {
return min(min(value.x, value.y), min(value.z, value.w));
}
void dataFilter_setValue(DATAFILTER_TYPE valueFromMin, DATAFILTER_TYPE valueFromMax, DATACATEGORY_TYPE category) {
if (filter_enabled) {

#ifdef DATAFILTER_ATTRIB
felixpalmer marked this conversation as resolved.
Show resolved Hide resolved
void dataFilter_setValue(DATAFILTER_TYPE valueFromMin, DATAFILTER_TYPE valueFromMax) {
if (filter_useSoftMargin) {
dataFilter_value = dataFilter_reduceValue(
smoothstep(filter_min, filter_softMin, valueFromMin) *
(1.0 - smoothstep(filter_softMax, filter_max, valueFromMax))
(1.0 - smoothstep(filter_softMax, filter_max, valueFromMax))
);
} else {
dataFilter_value = dataFilter_reduceValue(
step(filter_min, valueFromMin) * step(valueFromMax, filter_max)
);
}
}
#endif

#ifdef DATACATEGORY_ATTRIB
felixpalmer marked this conversation as resolved.
Show resolved Hide resolved
void dataFilter_setCategoryValue(DATACATEGORY_TYPE category) {
#if DATACATEGORY_CHANNELS == 1 // One 128-bit mask
int dataFilter_masks = filter_categoryBitMask[int(category / 32.0)];
int dataFilter_masks = filter_categoryBitMask[int(category / 32.0)];
#elif DATACATEGORY_CHANNELS == 2 // Two 64-bit masks
ivec2 dataFilter_masks = ivec2(
filter_categoryBitMask[int(category.x / 32.0)],
filter_categoryBitMask[int(category.y / 32.0) + 2]
);
ivec2 dataFilter_masks = ivec2(
filter_categoryBitMask[int(category.x / 32.0)],
filter_categoryBitMask[int(category.y / 32.0) + 2]
);
#elif DATACATEGORY_CHANNELS == 3 // Three 32-bit masks
ivec3 dataFilter_masks = filter_categoryBitMask.xyz;
ivec3 dataFilter_masks = filter_categoryBitMask.xyz;
#else // Four 32-bit masks
ivec4 dataFilter_masks = filter_categoryBitMask;
ivec4 dataFilter_masks = filter_categoryBitMask;
#endif

// Shift mask and extract relevant bits
DATACATEGORY_TYPE dataFilter_bits = DATACATEGORY_TYPE(dataFilter_masks) / pow(DATACATEGORY_TYPE(2.0), mod(category, 32.0));
dataFilter_bits = mod(floor(dataFilter_bits), 2.0);

#if DATACATEGORY_CHANNELS == 1
if(dataFilter_bits == 0.0) dataFilter_value = 0.0;
if(dataFilter_bits == 0.0) dataFilter_value = 0.0;
#else
if(any(equal(dataFilter_bits, DATACATEGORY_TYPE(0.0)))) dataFilter_value = 0.0;
#endif
} else {
dataFilter_value = 1.0;
}
}
#endif
`;

const fs = glsl`
Expand Down Expand Up @@ -163,15 +212,23 @@ function getUniforms64(opts?: DataFilterModuleSettings | {}): Record<string, any

const inject = {
'vs:#main-start': glsl`
#ifdef DATAFILTER_DOUBLE
dataFilter_setValue(
DATAFILTER_ATTRIB - filter_min64High + DATAFILTER_ATTRIB_64LOW,
DATAFILTER_ATTRIB - filter_max64High + DATAFILTER_ATTRIB_64LOW,
DATACATEGORY_ATTRIB
);
#else
dataFilter_setValue(DATAFILTER_ATTRIB, DATAFILTER_ATTRIB, DATACATEGORY_ATTRIB);
#endif
dataFilter_value = 1.0;
if (filter_enabled) {
#ifdef DATAFILTER_ATTRIB
#ifdef DATAFILTER_DOUBLE
dataFilter_setValue(
DATAFILTER_ATTRIB - filter_min64High + DATAFILTER_ATTRIB_64LOW,
DATAFILTER_ATTRIB - filter_max64High + DATAFILTER_ATTRIB_64LOW
);
#else
dataFilter_setValue(DATAFILTER_ATTRIB, DATAFILTER_ATTRIB);
#endif
#endif

#ifdef DATACATEGORY_ATTRIB
dataFilter_setCategoryValue(DATACATEGORY_ATTRIB);
#endif
}
`,

'vs:#main-end': glsl`
Expand Down