-
Notifications
You must be signed in to change notification settings - Fork 86
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #554 from brightics/brtc-issue-553
LGTM
- Loading branch information
Showing
4 changed files
with
316 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
198 changes: 198 additions & 0 deletions
198
function/python/brightics/function/textanalytics/meta/tfidf2.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,198 @@ | ||
| { | ||
| "script": { | ||
| "type": "", | ||
| "content": "" | ||
| }, | ||
| "specJson": { | ||
| "category": "textanalytics", | ||
| "func": "brightics.function.textanalytics$tfidf297577", | ||
| "name": "brightics.function.textanalytics$tfidf2", | ||
| "context": "python", | ||
| "label": "TF-IDF", | ||
| "description": "This is a function to calculate TF-IDF, abbreviated term for term frequency-inverse document frequency. \n\nReference:\n+ <https://en.wikipedia.org/wiki/Tf-idf>", | ||
| "tags": [], | ||
| "version": "3.6", | ||
| "inputs": { | ||
| "table": "" | ||
| }, | ||
| "outputs": { | ||
| "table_1": "", | ||
| "table_2": "", | ||
| "model": "" | ||
| }, | ||
| "meta": { | ||
| "table": { | ||
| "type": "table" | ||
| }, | ||
| "table_1": { | ||
| "type": "table" | ||
| }, | ||
| "table_2": { | ||
| "type": "table" | ||
| }, | ||
| "model": { | ||
| "type": "model" | ||
| } | ||
| }, | ||
| "params": [ | ||
| { | ||
| "id": "input_col", | ||
| "label": "Input Column", | ||
| "description": "", | ||
| "mandatory": true, | ||
| "items": [], | ||
| "visibleOption": [], | ||
| "control": "ColumnSelector", | ||
| "columnType": [ | ||
| "String", | ||
| "String[]" | ||
| ], | ||
| "validation": [], | ||
| "multiple": false | ||
| }, | ||
| { | ||
| "id": "max_df", | ||
| "label": "Maximum Document Frequency", | ||
| "description": "When building the vocabulary, ignore terms that have a document frequency strictly higher than the given threshold (corpus-specific stop words).", | ||
| "mandatory": false, | ||
| "items": [], | ||
| "visibleOption": [], | ||
| "control": "InputBox", | ||
| "columnType": [], | ||
| "validation": [], | ||
| "targetTable": [], | ||
| "placeHolder": "the number of documents", | ||
| "type": "Integer" | ||
| }, | ||
| { | ||
| "id": "min_df", | ||
| "label": "Minimum Document Frequency", | ||
| "description": "When building the vocabulary, ignore terms that have a document frequency strictly lower than the given threshold. This value is also called cut-off in the literature.", | ||
| "mandatory": false, | ||
| "items": [], | ||
| "visibleOption": [], | ||
| "control": "InputBox", | ||
| "columnType": [], | ||
| "validation": [], | ||
| "targetTable": [], | ||
| "placeHolder": "1 (value >= 0)", | ||
| "type": "Integer", | ||
| "min": 0 | ||
| }, | ||
| { | ||
| "id": "num_voca", | ||
| "label": "Number of Vocabularies", | ||
| "description": "The number of vocabularies that will be utilized to count their frequencies in the entire documents. It should be greater than or equal to two.", | ||
| "mandatory": false, | ||
| "items": [], | ||
| "visibleOption": [], | ||
| "control": "InputBox", | ||
| "columnType": [], | ||
| "validation": [], | ||
| "targetTable": [], | ||
| "placeHolder": "100 (value >= 2)", | ||
| "type": "Integer", | ||
| "min": 2 | ||
| }, | ||
| { | ||
| "id": "idf_weighting_scheme", | ||
| "label": "IDF Weighting Scheme", | ||
| "description": "Weighting scheme for IDF. Currently it is providing \"Unary\" and \"Inverse Document Frequency\" only.", | ||
| "mandatory": false, | ||
| "items": [ | ||
| { | ||
| "label": "Unary", | ||
| "value": "unary", | ||
| "default": false | ||
| }, | ||
| { | ||
| "label": "Inverse Document Frequency", | ||
| "value": "inverseDocumentFrequency", | ||
| "default": true | ||
| } | ||
| ], | ||
| "visibleOption": [], | ||
| "control": "RadioButton", | ||
| "columnType": [], | ||
| "validation": [], | ||
| "targetTable": [] | ||
| }, | ||
| { | ||
| "id": "norm", | ||
| "label": "Norm", | ||
| "description": "Norm used to normalize term vectors.", | ||
| "mandatory": false, | ||
| "items": [ | ||
| { | ||
| "label": "L1", | ||
| "value": "l1", | ||
| "default": false | ||
| }, | ||
| { | ||
| "label": "L2", | ||
| "value": "l2", | ||
| "default": true | ||
| } | ||
| ], | ||
| "visibleOption": [], | ||
| "control": "RadioButton", | ||
| "columnType": [], | ||
| "validation": [], | ||
| "targetTable": [] | ||
| }, | ||
| { | ||
| "id": "smooth_idf", | ||
| "label": "Smooth IDF", | ||
| "description": "Smooth idf weights by adding one to document frequencies, as if an extra document was seen containing every term in the collection exactly once. Prevents zero divisions.", | ||
| "mandatory": false, | ||
| "items": [], | ||
| "visibleOption": [], | ||
| "control": "BooleanRadio", | ||
| "columnType": [], | ||
| "validation": [], | ||
| "targetTable": [], | ||
| "defaultValue": true | ||
| }, | ||
| { | ||
| "id": "sublinear_tf", | ||
| "label": "Sublinear TF", | ||
| "description": "Apply sublinear tf scaling, i.e. replace \"tf\" with \"1 + log(tf)\".", | ||
| "mandatory": false, | ||
| "items": [], | ||
| "visibleOption": [], | ||
| "control": "BooleanRadio", | ||
| "columnType": [], | ||
| "validation": [], | ||
| "targetTable": [], | ||
| "defaultValue": false | ||
| }, | ||
| { | ||
| "id": "output_type", | ||
| "label": "Remove Zero Counts", | ||
| "description": "Delete zero counts.", | ||
| "mandatory": false, | ||
| "items": [], | ||
| "visibleOption": [], | ||
| "control": "BooleanRadio", | ||
| "columnType": [], | ||
| "validation": [], | ||
| "targetTable": [], | ||
| "defaultValue": true | ||
| }, | ||
| { | ||
| "id": "group_by", | ||
| "label": "Group By", | ||
| "description": "Columns to group by", | ||
| "mandatory": false, | ||
| "items": [], | ||
| "visibleOption": [], | ||
| "control": "ColumnSelector", | ||
| "columnType": [], | ||
| "validation": [], | ||
| "multiple": true, | ||
| "rowCount": 5 | ||
| } | ||
| ] | ||
| }, | ||
| "md": "" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters