dlatkInterface.py

#!/usr/bin/env python3

"""
Interface Module to DLATK
"""
print("\n")#clear some lines to make command more obvious
import os, getpass
import sys
import argparse
import time
import subprocess
from pprint import pprint
from numpy import isnan, sqrt, log
from configparser import SafeConfigParser
import gzip
from pathlib import Path

import dlatk.dlaWorker as dlaWorker
import dlatk.dlaConstants as dlac

import dlatk
from dlatk import DDLA
from dlatk.classifyPredictor import ClassifyPredictor
from dlatk.dimensionReducer import DimensionReducer, CCA
from dlatk.dlaWorker import DLAWorker
from dlatk.featureExtractor import FeatureExtractor
from dlatk.featureGetter import FeatureGetter
from dlatk.featureRefiner import FeatureRefiner
from dlatk.lexicainterface import lexInterface
from dlatk.mediation import MediationAnalysis
from dlatk.messageAnnotator import MessageAnnotator
from dlatk.messageTransformer import MessageTransformer
from dlatk.outcomeGetter import OutcomeGetter
from dlatk.outcomeAnalyzer import OutcomeAnalyzer
from dlatk.regressionPredictor import RegressionPredictor, CombinedRegressionPredictor, ClassifyToRegressionPredictor
from dlatk.semanticsExtractor import SemanticsExtractor
try:
    from dlatk.topicExtractor import TopicExtractor, LDAEstimator
except ImportError:
    print("Warning: Unable to import topic extractor; creating topics with LDA is disabled")

try:
    from dlatk.lib import wordcloud
except ImportError:
    print('warning: wordcloud not found.')

from dlatk.lexicainterface.lexInterface import LexInterfaceParser

def getInitVar(variable, parser, default, varList=False):
    if parser:
        if varList:
            return [o.strip() for o in parser.get('constants',variable).split(",")] if parser.has_option('constants',variable) else default
        else:
            return parser.get('constants',variable) if parser.has_option('constants',variable) else default
    else:
        return default


def colabify():

    dlatk_path = dlatk.__path__[0]
    subprocess.call(['bash', os.path.join(dlatk_path, 'tools', 'colabify.sh'), dlatk_path])

    return

#################################################################
### Main / Command-Line Processing:
##
#

def main(fn_args = None):
    """
    :param fn_args: string - ex "-d testing -t msgs -c user_id --add_ngrams -n 1 "
    """
    strTime = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
    dlac.warn("\n-----\nDLATK Interface Initiated: %s\n-----" % strTime)
    start_time = time.time()


    ##Argument Parser:
    init_parser = argparse.ArgumentParser(prefix_chars='-+', formatter_class=argparse.ArgumentDefaultsHelpFormatter, add_help=False)

    # Meta variables
    group = init_parser.add_argument_group('Meta Variables', '')
    group.add_argument('--lex_interface', dest='lexinterface', action='store_true', default=False,
                      help='Send arguments to lexInterface.py')
    group.add_argument('--to_file', dest='toinitfile', nargs='?', const=dlac.DEF_INIT_FILE, default=None,
                      help='write flag values to text file')
    group.add_argument('--from_file', type=str, dest='frominitfile', default='',
                       help='reads flag values from file')
    group.add_argument('--conf', '--mysql_config', '--mysql_config_file', metavar='HOST', dest='mysqlconfigfile', default=dlac.MYSQL_CONFIG_FILE,
                       help='Configuration file for MySQL connection settings (default: ~/.my.cnf or dlatk/lib/.dlatk.cnf)')
    group.add_argument('--colabify', dest='colabify', action='store_true', default=False, help='Flag for post-installtion Colab script')

    init_args, remaining_argv = init_parser.parse_known_args()
    
    if init_args.colabify:
        try:
            import google.colab
            colabify()
        except ImportError:
            print('warning: not a Colab environment.')
    
    if init_args.lexinterface:
        lex_parser = LexInterfaceParser(parents=[init_parser], mysql_config_file=init_args.mysqlconfigfile)
        lex_parser.processArgs(args=remaining_argv, parents=True)
        sys.exit()
    elif init_args.frominitfile:
        conf_parser = SafeConfigParser()
        conf_parser.read(init_args.frominitfile)
    else:
        conf_parser = None

    # Inherit options from init_parser
    parser = argparse.ArgumentParser(description='Extract and Manage Language Feature Data.',
        parents=[init_parser])

    group = parser.add_argument_group('Corpus Variables', 'Defining the data from which features are extracted.')

    group.add_argument('-e', '--engine', '--db_engine', metavar='DB', dest='dbengine', default=getInitVar('dbengine', conf_parser, dlac.DB_TYPE),
                        help='Database engine: mysql or sqlite.')
    group.add_argument('-d', '--corpdb', metavar='DB', dest='corpdb', default=getInitVar('corpdb', conf_parser, dlac.DEF_CORPDB),
                        help='Corpus Database Name.')
    group.add_argument('-t', '--corptable', metavar='TABLE', dest='corptable', default=getInitVar('corptable', conf_parser, dlac.DEF_CORPTABLE),
                        help='Corpus Table.')
    group.add_argument('-c', '-g', '--correl_field', '--group', '--group_by_field', metavar='FIELD', dest='correl_field',
                       default=getInitVar('correl_field', conf_parser, dlac.DEF_CORREL_FIELD),
                        help='Correlation Field (AKA Group Field): The field which features are aggregated over.')
    group.add_argument('--message_field', metavar='FIELD', dest='message_field', default=getInitVar('message_field', conf_parser, dlac.DEF_MESSAGE_FIELD),
                        help='The field where the text to be analyzed is located.')
    group.add_argument('--messageid_field', metavar='FIELD', dest='messageid_field', default=getInitVar('messageid_field', conf_parser, dlac.DEF_MESSAGEID_FIELD),
                        help='The unique identifier for the message.')
    group.add_argument('--date_field', metavar='FIELD', dest='date_field', default=getInitVar('date_field', conf_parser, dlac.DEF_DATE_FIELD),
                        help='Date a message was sent (if avail, for timex processing).')
    group.add_argument('--lexicondb', metavar='DB', dest='lexicondb', default=getInitVar('lexicondb', conf_parser, dlac.DEF_LEXICON_DB),
                        help='The database which stores all lexicons.')
    group.add_argument('--encoding', metavar='DB', dest='encoding', default=getInitVar('encoding', conf_parser, ''),
                        help='MySQL encoding')
    group.add_argument('--no_unicode', action='store_false', dest='useunicode', default=dlac.DEF_UNICODE_SWITCH,
                       help='Turn off unicode for reading/writing mysql and text processing.')
    group.add_argument('--language', '--lang', choices=['en', 'zh'], default=dlac.DEF_LANG,
                       help='Corpus language. Affects certain aspects of processing, like tokenization. Default: en.')

    group = parser.add_argument_group('Feature Variables', 'Use of these is dependent on the action.')
    group.add_argument('-f', '--feat_table', metavar='TABLE', dest='feattable', type=str, nargs='+', default=getInitVar('feattable', conf_parser, None, varList=True),
                       help='Table containing feature information to work with')
    group.add_argument('-n', '--set_n', metavar='N', dest='n', type=int, nargs='+', default=[dlac.DEF_N],
                       help='The n value used for n-grams or co-occurence features')
    group.add_argument('--no_metafeats', action='store_false', dest='metafeats', default=True,
                       help='indicate not to extract meta features (word, message length) with ngrams')
    group.add_argument('-l', '--lex_table', metavar='TABLE', dest='lextable', default=getInitVar('lextable', conf_parser, ''),
                       help='Lexicon Table Name: used for extracting category features from 1grams'+
                       '(or use --word_table to extract from other than 1gram)')
    group.add_argument('--word_table', metavar='WORDTABLE', dest='wordTable', default=getInitVar('wordTable', conf_parser, None),
                       help='Table that contains the list of words to give for lex extraction/group_freq_thresh')
    group.add_argument('--colloc_table', metavar='TABLE', dest='colloc_table', default=dlac.DEF_COLLOCTABLE,
                        help='Table that holds a list of collocations to be used as features.')
    group.add_argument('--colloc_column', metavar='COLUMN', dest='colloc_column', default=dlac.DEF_COLUMN_COLLOC,
                        help='Column giving collocations to be used as features.')
    group.add_argument('--create_collocation_scores', '--create_collocs', dest='createcollocscores', const=True, nargs='?', default=False,
                        help='Create ufeat table and annotate with pocc and npmi.')
    group.add_argument('--feature_type_name', metavar='STRING', dest='feature_type_name',
                        help='Customize the name of output features.')
    group.add_argument('--gzip_csv', metavar='filename', dest='gzipcsv', default='',
                       help='gz-csv filename used for extracting ngrams')
    group.add_argument('--categories', type=str, metavar='CATEGORY(IES)', dest='categories', nargs='+', default='',
                       help='Specify particular categories.')
    group.add_argument('--feat_blacklist', type=str, metavar='FEAT(S)', dest='feat_blacklist', nargs='+', default='',
                       help='Features to ban when correlating with outcomes.')
    group.add_argument('--feat_whitelist', type=str, metavar='FEAT(S)', dest='feat_whitelist', nargs='+', default='',
                       help='Only permit these features when correlating with outcomes (specify feature names or if feature table then read distinct features).')
    group.add_argument('--sqrt', action='store_const', dest='valuefunc', const=lambda d: sqrt(d),
                       help='square-roots normalized group_norm freq information.')
    group.add_argument('--log', action='store_const', dest='valuefunc', const=lambda d: log(d+1),
                       help='logs the normalized group_norm freq information.')
    group.add_argument('--anscombe', action='store_const', dest='valuefunc', const=lambda d: 2*sqrt(d+3/float(8)),
                       help='anscombe transforms normalized group_norm freq information.')
    group.add_argument('--boolean', action='store_const', dest='valuefunc', const=lambda d: float(1.0),
                       help='boolean transforms normalized group_norm freq information (1 if true).')
    group.add_argument('--lex_sqrt', action='store_const', dest='lexvaluefunc', const=lambda d: sqrt(d),
                       help='square-roots normalized group_norm lexicon freq information.')
    group.add_argument('--lex_log', action='store_const', dest='lexvaluefunc', const=lambda d: log(d+1),
                       help='logs the normalized group_norm lexicon freq information.')
    group.add_argument('--lex_anscombe', action='store_const', dest='lexvaluefunc', const=lambda d: 2*sqrt(d+3/float(8)),
                       help='anscombe transforms normalized group_norm lexicon freq information.')
    group.add_argument('--lex_boolean', action='store_const', dest='lexvaluefunc', const=lambda d: float(1.0),
                       help='boolean transforms normalized group_norm freq information (1 if true).')
    group.add_argument('--set_p_occ', '--set_pocc', metavar='P', dest='pocc', type=float, default=dlac.DEF_P_OCC,
                       help='The probability of occurence of either a feature or group (altnernatively if > 1, then limits to top p_occ features instead).')
    group.add_argument('--set_pmi_threshold', metavar='PMI', dest='pmi', type=float, default=dlac.DEF_PMI,
                       help='The threshold for the feat_colloc_filter.')
    group.add_argument('--set_min_feat_sum', metavar='N', dest='minfeatsum', type=int, default=dlac.DEF_MIN_FEAT_SUM,#TODO: Deprecate?
                       help='The minimum a feature must occur across all groups, to be kept.')
    group.add_argument('--topic_file', type=str, dest='topicfile', default='',
                       help='Name of topic file to use to build the topic lexicon.')
    group.add_argument('--num_topic_words', type=int, dest='numtopicwords', default=dlac.DEF_MAX_TOP_TC_WORDS,
                       help='Number of topic words to use as labels.')
    group.add_argument('--topic_lexicon', '--topic_lex', type=str, dest='topiclexicon', default='',
                       help='this is the (topic) lexicon name specified as part of --make_feat_labelmap_lex and --add_topiclex_from_topicfile')
    group.add_argument('--topic_list', type=str, dest='topiclist', default='', nargs='+',
                       help='this is the list of topics to group together in a plot for --feat_flexibin')
    group.add_argument('--topic_lex_method', type=str, dest='topiclexmethod', default=dlac.DEF_TOPIC_LEX_METHOD,
                       help='must be one of: "csv_lik", "standard"')
    group.add_argument('--weighted_lexicon', action='store_true', dest='weightedlexicon', default=False,
                       help='use with Extraction Action add_lex_table to make weighted lexicon features')
    group.add_argument('--num_bins', type=int, dest='numbins', default=None,
                       help='number of bins (feature refiner).')
    group.add_argument('--flexiplot_file', type=str, dest='flexiplotfile', default='',
                       help='use with Plot Action --feat_flexibin to specify a file to read for plotting')
    group.add_argument('--group_id_range', type=float, dest='groupidrange', nargs=2,
                       help='range of group id\'s to include in binning.')
    group.add_argument('--mask_table', type=str, metavar='TABLE', dest='masktable', default=None,
                       help='Table containing which groups run in various bins (for ttest).')
    group.add_argument('--embedding_model', '--emb_model', '--bert_model', type=str, metavar='NAME', dest='embmodel', default=dlac.DEF_EMB_MODEL,
                       help='Contextual Embedding model to use for extracting features.')
    group.add_argument('--emb_class', type=str, metavar='NAME', dest='embclass', default=None,
                       help='Contextual Embedding model class to use for extracting features.', choices=dlac.EMB_CLASS)
    group.add_argument('--tokenizer_model', type=str, dest='tokenizermodel', default=None,
                       help='Tokenizer model to use for tokenizing.')
    group.add_argument('--embedding_msg_aggregation', '--embedding_aggregations', '--emb_msg_aggregation', '--emb_aggregations', '--bert_msg_aggregation', '--bert_aggregations', 
                       type=str, metavar='AGG', nargs='+', dest='embaggs', default=dlac.DEF_EMB_AGGREGATION,
                       help='Aggregations to use with Contextual embedding model (e.g. mean, min, max).')
    group.add_argument('--embedding_layer_aggregation', '--layer_aggregation', '--bert_layer_aggregation', type=str, metavar='AGG', nargs='+', dest='emblayeraggs', default=dlac.DEF_EMB_LAYER_AGGREGATION,
                       help='Aggregations to use with Contextual embedding model (e.g. mean, min, max).')
    group.add_argument('--embedding_word_aggregation', '--word_aggregation', '--bert_word_aggregation', type=str, metavar='AGG', nargs='+', dest='transwordaggs', default=dlac.DEF_TRANS_WORD_AGGREGATION,
                       help='Aggregations to use for words (e.g. mean or concatenate).')
    group.add_argument('--embedding_layers', '--emb_layers', '--bert_layers', type=int, metavar='LAYER', nargs='+', dest='emblayers', default=dlac.DEF_EMB_LAYERS,
                       help='layers from Contextual model to keep.')
    group.add_argument('--embedding_no_context', '--emb_no_context', '--bert_no_context', action='store_true', dest='embnocontext', default=False,
                       help='encoded without considering context.')
    group.add_argument('--embedding_table_name', '--emb_table_name', dest='embtablename', default=None,
                       help='custom table name')                       
    group.add_argument('--batch_size', dest='batchsize', default=dlac.GPU_BATCH_SIZE, type=int, 
                       help='Specify the batch size for generating the embeddings.')
    group.add_argument('--embedding_keep_msg', '--emb_keep_msg', action='store_true', dest='embkeepmsg', default=False,
                       help='store embeddings as message_level feature table instead.')



    group = parser.add_argument_group('MySQL Interactoins', '')
    group.add_argument('--show_feature_tables', '--show_feat_tables', '--ls', action='store_true', dest='listfeattables', default=False,
                       help='List all feature tables for given corpdb, corptable and correl_field')
    group.add_argument('--show_tables', nargs='?', dest='showtables', default=False, const=True,
                       help='List all non-feature tables. Optional argument for "like"-style SQL call')
    group.add_argument('--describe_tables', '--desc_tables', nargs='*', dest='describetables', default=False,
                       help='')
    group.add_argument('--view_tables', '--view_data', nargs='*', dest='viewtables', default=False,
                       help='')
    group.add_argument('--create_random_sample', '--create_rand_sample', dest='createrandsample', default=None,
                       nargs='+', help='Given percentage, creates random sample of messages')
    group.add_argument('--copy_table', '--create_copied_table', dest='createcopiedtable', default=None,
                       nargs=2, help='OLD_TABLE NEW_TABLE copies OLD_TABLE to NEW_TABLE')
    group.add_argument('--extension', metavar='EXTENSION', dest='extension', default=None,
                       help='String added to the end of the feature table name')

    group.add_argument('--top_messages', type=int, dest='top_messages', nargs='?', const=dlac.DEF_TOP_MESSAGES, default=False,
                       help='Print top messages with the largest score for a given topic.')

    group = parser.add_argument_group('Outcome Variables', '')
    group.add_argument('--outcome_table', type=str, metavar='TABLE', dest='outcometable', default=getInitVar('outcometable', conf_parser, dlac.DEF_OUTCOME_TABLE),
                       help='Table holding outcomes (make sure correl_field type matches corpus\').')
    group.add_argument('--outcome_fields', '--outcomes',  type=str, metavar='FIELD(S)', dest='outcomefields', nargs='+', default=getInitVar('outcomefields', conf_parser, dlac.DEF_OUTCOME_FIELDS, varList=True),
                       help='Fields to compare with.')
    group.add_argument('--no_outcomes', action='store_const', const=[], dest='outcomefields',
                       help='Switch to override outcomes listed in init file.')
    group.add_argument('--outcome_controls', '--controls', type=str, metavar='FIELD(S)', dest='outcomecontrols', nargs='+', default=getInitVar('outcomecontrols', conf_parser, dlac.DEF_OUTCOME_CONTROLS, varList=True),
                       help='Fields in outcome table to use as controls for correlation(regression).')
    group.add_argument('--no_controls', action='store_const', const=[], dest='outcomecontrols',
                       help='Switch to override controls listed in init file.')
    group.add_argument('--categorical', '--categories_to_binary', '--cat_to_bin', type=str, metavar='FIELD(S)', dest='cattobinfields', nargs='+', default=[],
                       help='Fields with categorical variables to be transformed into a one hot representation')
    group.add_argument('--multiclass', '--categories_to_integer', '--cat_to_int', type=str, metavar='FIELD(S)', dest='cattointfields', nargs='+', default=[],
                       help='Fields with categorical variables to be transformed into a multiclass representation')
    group.add_argument('--outcome_interaction', '--interaction', type=str, metavar='TERM(S)', dest='outcomeinteraction', nargs='+', default=getInitVar('outcomeinteraction', conf_parser, dlac.DEF_OUTCOME_CONTROLS, varList=True),
                       help='Fields in outcome table to use as controls and interaction terms for correlation(regression).')
    group.add_argument('--fold_column', '--fold_labels', type=str, dest='fold_column', default=None,
                       help='Fields in outcome table to use as labels for prespecified folds in classification/regression cross-validation.')
    group.add_argument('--test_folds', type=int, dest='test_folds', nargs='+', default=None,
                       help='Limit tests of classification/regression cross-validation to these folds.')
    group.add_argument('--feat_names', type=str, metavar='FIELD(S)', dest='featnames', nargs='+', default=getInitVar('featnames', conf_parser, dlac.DEF_FEAT_NAMES, varList=True),
                       help='Limit outputs to the given set of features.')
    group.add_argument("--group_freq_thresh", type=int, metavar='N', dest="groupfreqthresh", default=getInitVar('groupfreqthresh', conf_parser, None),
                       help="minimum WORD frequency per correl_field to include correl_field in results")
    group.add_argument('--output_dir', type=str, dest='outputdir', default=dlac.DEF_OUTPUT_DIR, help='folder to store the outputs, use either this or --output_name')
    group.add_argument('--output_name', '--output', '--output_file', type=str, dest='outputname', default=getInitVar('outputname', conf_parser, ''),
                       help='overrides the default filename for output')
    group.add_argument('--max_wordcloud_words', '--max_tagcloud_words', type=int, metavar='N', dest='maxtcwords', default=dlac.DEF_MAX_TC_WORDS,
                       help='Max words to appear in a tagcloud')
    group.add_argument('--show_feat_freqs', action='store_true', dest='showfeatfreqs', default=dlac.DEF_SHOW_FEAT_FREQS,)
    group.add_argument('--not_show_feat_freqs', action='store_false', dest='showfeatfreqs', default=dlac.DEF_SHOW_FEAT_FREQS,
                       help='show / dont show feature frequencies in output.')
    group.add_argument('--tagcloud_filter', '--wordcloud_filter', action='store_true', dest='tcfilter', default=dlac.DEF_TC_FILTER,)
    group.add_argument('--no_tagcloud_filter', '--no_wordcloud_filter', action='store_false', dest='tcfilter', default=dlac.DEF_TC_FILTER,
                       help='filter / dont filter tag clouds for duplicate info in phrases.')
    group.add_argument('--feat_labelmap_table', type=str, dest='featlabelmaptable', default=getInitVar('featlabelmaptable', conf_parser, ''),
                       help='specifies an lda mapping tablename to be used for LDA topic mapping')
    group.add_argument('--feat_labelmap_lex', type=str, dest='featlabelmaplex', default=getInitVar('featlabelmaplex', conf_parser, ''),
                       help='specifies a lexicon tablename to be used for the LDA topic mapping')
    group.add_argument('--bracket_labels', action='store_true', dest='bracketlabels', default='',
                       help='use with: feat_labelmap_lex... if used, the labelmap features will be contained within brackets')
    group.add_argument('--comparative_tagcloud', '--comparative_wordcloud', action='store_true', dest='compTagcloud', default=False,
                       help='used with --sample1 and --sample2, this option uses IDP to compare feature usage')
    group.add_argument('--sample1', type=str, nargs='+', dest="compTCsample1", default=[],
                       help='first sample of group to use in comparison [use with --comparative_tagcloud]'+
                       "(use * to mean all groups in featuretable)")
    group.add_argument('--sample2', type=str, nargs='+', dest="compTCsample2", default=[],
                       help='second sample of group to use in comparison [use with --comparative_tagcloud]'+
                       "(use * to mean all groups in featuretable)")
    group.add_argument('--csv', action='store_true', dest='csv',
                       help='generate csv correl matrix output as well')
    group.add_argument('--pickle', action='store_true', dest='pickle',
                       help='generate pickle of the correl matrix output as well')
    group.add_argument('--sort', action='store_true', dest='sort',
                       help='add sorted output for correl matrix')
    group.add_argument('--whitelist', action='store_true', dest='whitelist', default=False,
                       help='Uses feat_whitelist or --lex_table and --categories.')
    group.add_argument('--blacklist', action='store_true', dest='blacklist', default=False,
                       help='Uses feat_blacklist or --lex_table and --categories.')
    group.add_argument('--spearman', action='store_true', dest='spearman',
                       help='Use Spearman R instead of Pearson.')
    group.add_argument('--logistic_reg', '--logistic', '--logistic_regression', action='store_true', dest='logisticReg', default=False,
                       help='Use logistic regression instead of linear regression. This is better for binary outcomes.')
    group.add_argument('--cohens_d', action='store_true', dest='cohensd', default=False,
                       help='Report Cohen\'s D with logistic regression instead coefficients (B\'s).')
    group.add_argument('--IDP', '--idp', action='store_true', dest='IDP', default=False,
                       help='Use IDP instead of linear regression/correlation [only works with binary outcome values]')
    group.add_argument('--AUC', '--auc', action='store_true', dest='auc', default=False,
                       help='Use AUC instead of linear regression/correlation [only works with binary outcome values]')
    group.add_argument('--zScoreGroup', action='store_true', dest='zScoreGroup', default=False,
                       help="Outputs a certain group's zScore for all feats, which group is determined by the boolean outcome value [MUST be boolean outcome]")
    group.add_argument('--p_correction', metavar='METHOD', type=str, dest='p_correction_method', default=getInitVar('p_correction_method', conf_parser, dlac.DEF_P_CORR),
                       help='Specify a p-value correction method: simes, holm, hochberg, hommel, bonferroni, BH, BY, fdr, none',
                       choices=dlac.DEF_P_MAPPING.keys())
    group.add_argument('--no_bonferroni', action='store_false', dest='bonferroni', default=True,
                       help='Turn off bonferroni correction of p-values.')
    group.add_argument('--no_correction', action='store_const', const='', dest='p_correction_method',
                       help='Turn off BH correction of p-values.')
    group.add_argument('--nvalue', type=bool, dest='nvalue', default=True,
                       help='Report n values.')
    group.add_argument('--conf_int', type=bool, dest='confint', default=True,
                       help='Report confidence intervals.')
    group.add_argument('--freq', type=bool, dest='freq', default=True,
                       help='Report freqs.')
    group.add_argument('--tagcloud_colorscheme', '--wordcloud_colorscheme', type=str, dest='tagcloudcolorscheme', default=getInitVar('tagcloudcolorscheme', conf_parser, 'multi'),
                       help='specify a color scheme to use for tagcloud generation. Default: multi, also accepts red, blue, red-random, redblue, bluered')
    group.add_argument('--clean_cloud', action='store_true', dest='cleancloud', default=False,
                       help='Replaces characters in the middle of explatives/slurs with ***. ex: f**k')
    group.add_argument('--weighted_sample', dest='weightedsample', default=dlac.DEF_WEIGHTS,
                       help='Field in outcome table to use as weights for correlation(regression).')
    group.add_argument('--keep_low_variance_outcomes', '--keep_low_variance', dest='low_variance_thresh', action='store_false', default=dlac.DEF_LOW_VARIANCE_THRESHOLD,
                       help='Does not remove low variance outcomes and controls from analysis')
    group.add_argument("--min_abs_effect_size", "--min_es", type=float, metavar='r', dest="minabseffectsize", default = 0.0,
                       help="Minimum effect size to add to tagcloud. Default = 0.0.")



    group.add_argument('--interactions', action='store_true', dest='interactions', default=False,
                       help='Includes interaction terms in multiple regression.')
    group.add_argument('--bootstrapp', '--bootstrap', dest='bootstrapp', type=int, default = 0,
                       help="Bootstrap p-values (only works for AUCs for now) ")
    group.add_argument("--p_value", type=float, metavar='P', dest="maxP", default = getInitVar('maxP', conf_parser, float(dlac.DEF_P)),
                       help="Significance threshold for returning results. Default = 0.05.")
    group.add_argument("--where", type=str, dest="groupswhere", default = '',
                       help="Filter groups with sql-style call. ")

    group = parser.add_argument_group('Mediation Variables', '')
    group.add_argument('--mediation', action='store_true', dest='mediation', default=False,
                       help='Run mediation analysis.')
    group.add_argument('--mediation_bootstrap', '--mediation_boot', action='store_true', dest='mediationboot', default=False,
                       help='Run mediation analysis with bootstrapping. The parametric (non-bootstrapping) method is default.')
    group.add_argument("--mediation_boot_num", type=int, metavar='N', dest="mediationbootnum", default = int(dlac.DEF_MEDIATION_BOOTSTRAP),
                       help="The number of repetitions to run in bootstrapping with mediation analysis. Default = 1000.")
    group.add_argument('--outcome_pathstarts', '--path_starts', type=str, metavar='FIELD(S)', dest='outcomepathstarts', nargs='+', default=dlac.DEF_OUTCOME_PATH_STARTS,
                       help='Fields in outcome table to use as treatment in mediation analysis.')
    group.add_argument('--outcome_mediators', '--mediators', type=str, metavar='FIELD(S)', dest='outcomemediators', nargs='+', default=dlac.DEF_OUTCOME_MEDIATORS,
                       help='Fields in outcome table to use as mediators in mediation analysis.')
    group.add_argument('--feat_as_path_start', action='store_true', dest='feat_as_path_start', default=False,
                       help='Use path start variables located in a feature table. Used in mediation analysis.')
    group.add_argument('--feat_as_outcome', action='store_true', dest='feat_as_outcome', default=False,
                       help='Use outcome variables located in a feature table. Used in mediation analysis.')
    group.add_argument('--feat_as_control', action='store_true', dest='feat_as_control', default=False,
                       help='Use control variables located in a feature table. Used in mediation analysis.')
    group.add_argument('--no_features', action='store_true', dest='no_features', default=False,
                       help='All mediation analysis variables found corptable. No feature table needed.')
    group.add_argument('--mediation_csv', action='store_true', dest='mediationcsv', default=False,
                       help='Print results to a CSV. Default file name is mediation.csv. Use --output_name to specify file name.')
    group.add_argument('--mediation_no_summary', action='store_false', dest='mediationsummary', default=True,
                       help='Print results to a CSV. Default file name is mediation.csv. Use --output_name to specify file name.')
    group.add_argument('--mediation_method', metavar='METHOD', type=str, dest='mediation_style', default='baron',
                       help='Specify a mediation method: baron, imai, both')


    group = parser.add_argument_group('Prediction Variables', '')
    group.add_argument('--adapt_tables', metavar='TABLE_NUM', dest='adapttable', type=int, nargs='+', default=getInitVar('adapttable', conf_parser, None, varList=True),
                       help='NOT IMPLEMENTED: Table(s) containing feature information to be adapted')
    group.add_argument('--adapt_control_names', metavar='COLUMN', dest='adaptcolumns', type=str, nargs='+', default=None,
                        help='NOT IMPLEMENTED: Controls to be used for adaptation.')
    group.add_argument('--model', type=str, metavar='name', dest='model', default=getInitVar('model', conf_parser, dlac.DEF_MODEL),
                       help='Model to use when predicting: svc, linear-svc, ridge, linear.')
    group.add_argument('--turn_off_backoff_model', action='store_true', dest='turn_off_backoff_model', default=False,
                       help='Turn off backoff model (logistic or linear) when using a small number of features (uses --model instead)')
    group.add_argument('--combined_models', type=str, nargs='+', metavar='name', dest='combmodels', default=dlac.DEF_COMB_MODELS,
                       help='Model to use when predicting: svc, linear-svc, ridge, linear.')
    group.add_argument('--sparse', action='store_true', dest='sparse', default=False,
                       help='use sparse representation for X when training / testing')
    group.add_argument('--folds', type=int, metavar='NUM', dest='folds', default=dlac.DEF_FOLDS,
                       help='Number of folds for functions that run n-fold cross-validation')
    group.add_argument('--outlier_to_mean', '--outliers_to_mean', dest='outlier_to_mean', nargs='?', type=float, default=False, const=dlac.DEF_OUTLIER_THRESHOLD,
                        help='')
    group.add_argument('--picklefile', type=str, metavar='filename', dest='picklefile', default='',
                       help='Name of file to save or load pickle of model')
    group.add_argument('--all_controls_only', action='store_true', dest='allcontrolsonly', default=True,
                       help='Only uses all controls when prediction doing test_combo_regression')
    group.add_argument('--all_control_combinations', action='store_false', dest='allcontrolsonly', default=True,
                       help='Run all control combinations when prediction doing test_combo_regression')
    group.add_argument('--no_lang', action='store_true', dest='nolang', default=False,
                       help='Runs with language features excluded')
    group.add_argument('--control_combo_sizes', '--combo_sizes', type=int, metavar="index", nargs='+', dest='controlcombosizes',
                       default=[], help='specify the sizes of control combos to use')
    group.add_argument('--residualized_controls', '--res_controls', action='store_true', dest='res_controls', default=False,
                       help='Finds residuals for controls and tries to predict beyond them (only for combo test)')
    group.add_argument('--prediction_csv', '--pred_csv', action='store_true', dest='pred_csv',
                       help='write yhats in a separate csv')
    group.add_argument('--probability_csv', '--prob_csv', action='store_true', dest='prob_csv',
                       help='write probabilities for yhats in a separate csv')
    group.add_argument('--weighted_eval', type=str, dest='weightedeval', default=None,
                       help='Column to weight the evaluation.')
    group.add_argument('--no_standardize', action='store_false', dest='standardize', default=True,
                       help='turn off standardizing variables before prediction')
    group.add_argument('--feature_selection', '--feat_selection', metavar='NAME', type=str, dest='featureselection', default=getInitVar('featureselection', conf_parser, ''),
                       help='Specify feature selection pipeline in prediction: magic_sauce, univariateFWE, PCA.')
    group.add_argument('--feature_selection_string', metavar='NAME', type=str, dest='featureselectionstring', default=getInitVar('featureselectionstring', conf_parser, ''),
                       help='Specify any feature selection pipeline in prediction.')
    group.add_argument('--train_bootstraps', metavar='NUM_ITER', dest='train_bootstraps', type=int, default=None, 
                       help='Number of bootstrapped training resamples to use (default None -- no bootstrap).')
    group.add_argument('--train_bootstraps_ns', metavar='N N N...', dest='train_bootstraps_ns', nargs='+', type=int, default=None, 
                       help='Various sample sizes to use while doing bootstrapped training.')


    group = parser.add_argument_group('Standard Extraction Actions', '')
    group.add_argument('--add_ngrams', action='store_true', dest='addngrams',
                       help='add an n-gram feature table. (uses: n, can flag: sqrt), gzip_csv'
                       'can be used with or without --use_collocs')
    group.add_argument('--add_ngrams_from_tokenized', action='store_true', dest='addngramsfromtok',
                       help='add an n-gram feature table from a tokenized table. Table must be JSON list of tokens.'
                       '(uses: n, can flag: sqrt), gzip_csv.')
    group.add_argument('--use_collocs', action='store_true', dest='use_collocs',
                       help='when extracting ngram features, use a table of collocations to parse text into ngrams'
                            'by default does not include subcollocs, this can be changed with the --include_sub_collocs option ')
    group.add_argument('--include_sub_collocs', action='store_true', dest='include_sub_collocs',
                       help='count all sub n-grams of collocated n-grams'
                            'if "happy birthday" is designated as a collocation, when you see "happy birthday" in text'
                            'count it as an instance of "happy", "birthday", and "happy birthday"')
    group.add_argument('--colloc_pmi_thresh', metavar="PMI", dest='colloc_pmi_thresh', type=float, default=dlac.DEF_PMI,
                       help='The PMI threshold for which multigrams from the colloctable to conscider as valid collocs'
                            'looks at the feat_colloc_filter column of the specified colloc table')

    group.add_argument('--add_char_ngrams', action='store_true', dest='addcharngrams',
                       help='add a character n-gram feature table. (uses: n, can flag: sqrt), gzip_csv'
                       'can be used with or without --use_collocs')
    group.add_argument('--no_lower', action='store_false', dest='lowercaseonly', default=dlac.LOWERCASE_ONLY,
                       help='')
    group.add_argument('--add_lex_table', action='store_true', dest='addlextable',
                       help='add a lexicon-based feature table. (uses: l, weighted_lexicon, can flag: anscombe).')
    group.add_argument('--add_corp_lex_table', action='store_true', dest='addcorplextable',
                       help='add a lexicon-based feature table based on corpus. (uses: l, weighted_lexicon, can flag: anscombe).')
    group.add_argument('--add_phrase_table', action='store_true', dest='addphrasetable',
                       help='add constituent and phrase feature tables. (can flag: sqrt, anscombe).')
    group.add_argument('--add_pos_table', action='store_true', dest='addpostable',
                       help='add pos feature tables. (can flag: sqrt, anscombe).')
    group.add_argument('--add_pos_ngram_table', action='store_true', dest='pos_ngram',
                       help='add pos with ngrams feature table. (can flag: sqrt, anscombe).')
    group.add_argument('--add_lda_table', metavar='LDA_MSG_TABLE', dest='addldafeattable',
                       help='add lda feature tables. (can flag: sqrt, anscombe).')
    group.add_argument('--print_tokenized_lines', metavar="FILENAME", dest='printtokenizedlines', default = None,
                       help='prints tokenized version of messages to lines.')
    group.add_argument('--print_joined_feature_lines', metavar="FILENAME", dest='printjoinedfeaturelines', default = None,
                       help='prints feature table with line per group joined by spaces (with MWEs joined by underscores) for import into Mallet.')
    group.add_argument('--add_topiclex_from_topicfile', action='store_true', dest='addtopiclexfromtopicfile',
                       help='creates a lexicon from a topic file, requires --topic_file --topic_lexicon --lex_thresh --topic_lex_method')
    group.add_argument('--add_timexdiff', action='store_true', dest='addtimexdiff',
                       help='extract timeex difference features (mean and std) per group.')
    group.add_argument('--add_postimexdiff', action='store_true', dest='addpostimexdiff',
                       help='extract timeex difference features and POS tags per group.')
    group.add_argument('--add_wn_nopos', action='store_true', dest='addwnnopos',
                       help='extract WordNet concept features (not considering POS) per group.')
    group.add_argument('--add_wn_pos', action='store_true', dest='addwnpos',
                       help='extract WordNet concept features (considering POS) per group.')
    group.add_argument('--add_fleschkincaid', '--add_fkscore', action='store_true', dest='addfkscore',
                       help='add flesch-kincaid scores, averaged per group.')
    group.add_argument('--add_pnames', type=str, nargs=2, dest='addpnames',
                       help='add an people names feature table. (two agrs: NAMES_LEX, ENGLISH_LEX, can flag: sqrt)')
    group.add_argument('--add_embedding', '--add_emb_feat',  '--add_bert', action='store_true', dest='embaddfeat',
                       help='add BERT mean features (optionally add min, max, --bert_model large)')
    group.add_argument('--lexicon_normalization', '--lex_norm', '--dict_norm', action='store_true', help='Use weighting over lexicon terms (instead of over all terms).')
    group.add_argument('--multicategory_normalization', '--liwc_normalization', '--liwc_norm', action='store_true', help='Use weighting over lexicon terms across terms in all categories. Similar to --lexicon_normalization but totals are across all lexicon categories.')


    group = parser.add_argument_group('Messages Transformation Actions', '')
    group.add_argument('--add_tokenized', action='store_true', dest='addtokenized',
                       help='adds tokenized version of message table.')
    group.add_argument('--add_sent_tokenized', action='store_true', dest='addsenttokenized',
                       help='adds sentence tokenized version of message table.')
    group.add_argument('--add_sent_per_row', action='store_true', dest='addsentperrow',
                       help='adds sentence tokenized version of message table with each sentence as a row in MySQL table.')
    group.add_argument('--add_parses', action='store_true', dest='addparses',
                       help='adds parsed versions of message table.')
    group.add_argument('--add_segmented', action="store_true", dest='addsegmented', default=False,
                       help='adds segmented versions of message table.')
    group.add_argument('--segmentation_model',type=str, dest='segmentationModel', default="ctb",
                       help='Chooses which model to use for message segmentation (CTB or PKU; Default CTB)')
    group.add_argument('--add_tweettok', action='store_true', dest='addtweettok',
                       help='adds tweetNLP tokenized versions of message table.')
    group.add_argument('--add_tweetpos', action='store_true', dest='addtweetpos',
                       help='adds tweetNLP pos tagged versions of message table.')
    group.add_argument('--add_lda_messages', metavar='LDA_States_File', dest='addldamsgs',
                       help='add lda topic version of message table.')
    group.add_argument('--add_outcome_feats', action='store_true', dest='addoutcomefeats',
                       help='add a feature table from the specified outcome table.')

    group = parser.add_argument_group('Message Cleaning Actions', '')
    group.add_argument('--language_filter', '--lang_filter',  type=str, metavar='FIELD(S)', dest='langfilter', nargs='+', default=[],
                       help='Filter message table for list of languages.')
    group.add_argument('--light_english_filter', dest='lightenglishfilter', action = 'store_true', help="Apply a less stringent English filter than --language_filter")
    group.add_argument('--clean_messages', dest='cleanmessages', action = 'store_true', help="Remove URLs, hashtags and @ mentions from messages")
    group.add_argument('--deduplicate', action='store_true', dest='deduplicate',
                       help='Removes duplicate messages within correl_field grouping, writes to new table corptable_dedup Not to be run at the message level.')
    group.add_argument('--spam_filter', dest='spamfilter', metavar="SPAM_THRESHOLD", type=float, nargs='?', const=dlac.DEF_SPAM_FILTER,
                       help='Removes users (by correl_field grouping) with percentage of spam messages > threshold, writes to new table corptable_nospam '
                       'with new column is_spam. Defaul threshold = %s'%dlac.DEF_SPAM_FILTER)

    group = parser.add_argument_group('LDA Actions', '')
    group.add_argument('--add_message_id', type=str, nargs=2, dest='addmessageid',
                       help='Adds the message IDs to the topic distributions and stores the result in --output_name. '
                            'Previously addMessageID.py (two args: MESSAGE_FILE, STATE_FILE)')
    group.add_argument('-m', '--lda_msg_tbl', metavar='TABLE', dest='ldamsgtbl', type=str, default=dlac.DEF_LDA_MSG_TABLE,
                           help='LDA Message Table')
    group.add_argument('--create_dists', action='store_true', dest='createdists',
                           help='Create conditional prob, and likelihood distributions.')

    group = parser.add_argument_group('Semantic Extraction Actions', '')
    group.add_argument('--add_ner', action='store_true', dest='addner',
                       help='extract ner features from xml files (corptable should be directory of xml files).')


    group = parser.add_argument_group('Feature Table Aanalyses', '')
    group.add_argument('--ttest_feat_tables', action='store_true', dest='ttestfeats',
                       help='Performs ttest on differences between group norms for 2 tables, within features')

    group = parser.add_argument_group('Refinement Actions', '')
    group.add_argument('--feat_occ_filter', action='store_true', dest='featoccfilter',
                       help='remove infrequent features. (uses variables feat_table and p_occ).')
    group.add_argument('--combine_feat_tables', '--combine_feats', type=str, dest='combinefeattables', default=None,
                       help='Given multiple feature table, combines them (provide feature name) ')
    group.add_argument('--add_feat_norms', action='store_true', dest='addfeatnorms',
                       help='calculates and adds the mean normalized (feat_norm) value for each row (uses variable feat_table).')
    group.add_argument('--add_standardized_feats', '--add_std_feats', action='store_true', dest='addstdfeats',
                       help='Adds a copy of the feature table where group_norms have been standardized (new table appended with "z").')


    group.add_argument('--feat_colloc_filter', action='store_true', dest='featcollocfilter',
                       help='removes featrues that do not pass as collocations. (uses feat_table).')
    group.add_argument('--feat_correl_filter', action='store_true', dest='featcorrelfilter',
                       help='removes features that do not pass correlation sig tests with given outcomes (uses -f --outcome_table --outcomes).')
    group.add_argument('--make_topic_labelmap_lex', action='store_true', dest='maketopiclabelmap', default=False,
                       help='Makes labelmap lexicon from topics. Requires --topic_lexicon, --num_topic_words. Optional: --weighted_lexicon')
    group.add_argument('--tf_idf', action='store_true', dest='tfidf', default=False,
                       help='Given an ngram feature table, creates a new feature table with tf-idf (uses -f).')
    group.add_argument('--feat_group_by_outcomes', action='store_true', dest='featgroupoutcomes', default=False,
                       help='Creates a feature table grouped by a given outcome (requires outcome field, can use controls)')
    group.add_argument('--aggregate_feats_by_new_group', action='store_true', dest='aggregategroup', default=False,
                       help='Aggregate feature table by group field (i.e. message_id features by user_ids). Specify new group with --group_by field; old group is whatever was used for the feature table.')

    group.add_argument('--feat_archetype_similarity', '--archtype_similarity', type=str, dest='archetypesimilarity', \
                       default=None, help='Creates a feature table of archetype similarities (provide archetype feature table)')
    
    group.add_argument('--interpolate_aggregated_feats', '--interpolate_feats', type=float, dest='interpolategroup', default=None,
                       help='Aggregates features from a lower level to new group by field, interpolating across specified amount of days.')



    group = parser.add_argument_group('Outcome Actions', '')
    group.add_argument('--print_csv', metavar="FILENAME", dest='printcsv', default = None,
                       help='prints group normalized values use for correlation.')
    group.add_argument('--print_freq_csv', metavar="FILENAME", dest='printfreqcsv', default = None,
                       help='prints frequencies instead of group normalized values')
    group.add_argument('--print_numgroups', action='store_true', dest='printnumgroups', default = False,
                       help='prints number of groups per outcome field')
    group.add_argument('--densify_table', dest='densifytable', nargs=3, default = None,
                       help='Create a dense csv given a db, table, and three columns. Three variables needed: ROW COL VALUE')

    group = parser.add_argument_group('Correlation Actions', 'Finds one relationship at a time (but can still adjust for others)')
    group.add_argument('--correlate', '--dla', action='store_true', dest='correlate',
                       help='correlate with outcome (uses variable feat_table and all outcome variables).')
    group.add_argument('--rmatrix', action='store_true', dest='rmatrix',
                       help='output a correlation matrix to a file in the output dir.')
    group.add_argument('--combo_rmatrix', action='store_true', dest='combormatrix',
                       help='output a correlation matrix with all combinations of controls.')
    group.add_argument('--topic_dupe_filter', action='store_true', dest='topicdupefilter',
                       help='remove topics not passing a duplicate filter from the correlation matrix')
    group.add_argument('--tagcloud', '--wordcloud', action='store_true', dest='tagcloud',
                       help='produce data for making wordle tag clouds (same variables as correlate).')
    group.add_argument('--topic_tagcloud', '--topic_wordcloud', action='store_true', dest='topictc',
                       help='produce data for making topic wordles (must be used with a topic-based feature table and --topic_lexicon).')
    group.add_argument('--corp_topic_tagcloud', '--corp_topic_wordcloud', action='store_true', dest='corptopictc',
                       help='produce data for making topic wordles (must be used with a topic-based feature table and --topic_lexicon).')
    group.add_argument('--make_wordclouds', '--make_tagclouds', action='store_true', dest='makewordclouds',
                       help="make wordclouds from the output tagcloud file.")
    group.add_argument('--make_topic_wordclouds', '--make_topic_tagclouds', action='store_true', dest='maketopicwordclouds',
                       help="make topic wordclouds, needs an output topic tagcloud file.")
    group.add_argument('--make_all_topic_wordclouds', '--make_all_topic_tagclouds', action='store_true', dest='makealltopicwordclouds',
                       help="make all topic wordclouds for a given topic lexicon.")
    group.add_argument('--keep_duplicates', action='store_true', dest='keepduplicates',
                       help="Create topic wordclouds for duplicate filtered topics.")
    group.add_argument('--use_featuretable_feats', action='store_true', dest='useFeatTableFeats',
                       help='use 1gram table to be used as a whitelist when plotting')
    group.add_argument('--outcome_with_outcome', action='store_true', dest='outcomeWithOutcome',
                       help="correlates all outcomes in --outcomes with each other in addition to the features")
    group.add_argument('--outcome_with_outcome_only', action='store_true', dest='outcomeWithOutcomeOnly',
                       help="correlates all outcomes in --outcomes with each other in WITHOUT the features")
    group.add_argument('--output_interaction_terms', action='store_true', dest='outputInteractionTerms',
                       help='with this flag, outputs the coefficients from the interaction terms as r values '+
                       'the outcome coefficients. Use with --outcome_interaction FIELD1 [FIELD2 ...]')
    group.add_argument('--interaction_ddla', dest='interactionDdla',
                       help="column name from the outcome table that is going to be used in DDLA:"+
                       "First, finding terms with significant interaction, then taking correlations for groups with outcome = 1 and = 0 separately")
    group.add_argument('--interaction_ddla_pvalue', dest='ddlaSignificance', type=float,default=0.001,
                       help="Set level of significance to filter ddla features by")
    group.add_argument('--DDLA', dest='ddlaFiles', nargs=2, help="Compares two csv's that have come out of DLA. Requires --freq and --nvalue to have been used")
    group.add_argument('--DDLATagcloud', '--DDLAWordcloud', dest='ddlaTagcloud', action='store_true',
                       help="Makes a tagcloud file from the DDLA output. Uses deltaR as size, r_INDEX as color. ")

    group = parser.add_argument_group('Multiple Regression Actions', 'Find multiple relationships at once')
    group.add_argument('--multir', action='store_true', dest='multir',
                       help='multivariate regression with outcome (uses variable feat_table and all outcome variables, optionally csv).')

    group = parser.add_argument_group('Prediction Actions', '')
    group.add_argument('--train_regression', '--train_reg', action='store_true', dest='trainregression', default=False,
                       help='train a regression model to predict outcomes based on feature table')
    group.add_argument('--test_regression', action='store_true', dest='testregression', default=False,
                       help='train/test a regression model to predict outcomes based on feature table')
    group.add_argument('--nfold_regression', '--combo_test_regression', '--combo_test_reg', '--nfold_test_regression', action='store_true', dest='combotestregression', default=False,
                       help='train/test a regression model with and without all combinations of controls')
    group.add_argument('--predict_regression', '--predict_reg', action='store_true', dest='predictregression', default=False,
                       help='predict outcomes based on loaded or trained regression model')
    group.add_argument('--control_adjust_outcomes_regression', '--control_adjust_reg', action='store_true', default=False,  dest='controladjustreg',
                       help='predict outcomes from controls and produce adjusted outcomes')
    group.add_argument('--test_combined_regression', type=str, metavar="featuretable", nargs='+', dest='testcombregression', default=[],
                       help='train and test combined model (must specify at least one addition feature table here)')
    group.add_argument('--predict_regression_to_feats', type=str, dest='predictrtofeats', default=None,
                       help='predict outcomes into a feature file (provide a name)')
    group.add_argument('--predict_regression_to_outcome_table', type=str, dest='predictRtoOutcomeTable', default=None,
                       help='predict outcomes into an outcome table (provide a name)')
    group.add_argument('--predict_cv_to_feats', '--predict_combo_to_feats', '--predict_regression_all_to_feats', type=str, dest='predictalltofeats', default=None,
                       help='predict outcomes into a feature file (provide a name)')

    group = parser.add_argument_group('Factor Adaptation Actions', '')
    group.add_argument('--fs_params', action='store_true', dest='fsparams', default=False, help = 'send values for feature selection parameters')
    group.add_argument('--k_best', type=str, dest='kbest', nargs='+', help='vaiables needed for feature selection .')
    group.add_argument('--pca_comp', type=str, dest='pcacomp', nargs='+', help='vaiables needed for feature selection .')
    group.add_argument('--adaptation_factors', '--factors', type=str, metavar='FIELD(S)', dest='adaptationfactors', nargs='+', help='Fields in outcome table to use as factors for adaptation in FA or RFA.')
    group.add_argument('--factor_selection_type', type=str , dest = 'factorselectiontype', default='rfe', help='chooses the type of factor selection, either pca or rfe.')
    group.add_argument('--num_of_factors', type=int, dest='numoffactors', nargs='+',
                       help='specifies the number of factors for factor selection. 0 means all factor1. ')
    group.add_argument('--paired_factors',  action='store_true',  dest = 'pairedfactors', default=False , help='multiplying factors to themself, to make bigger pool of factors')
    group.add_argument('--report', action='store_true', dest='report', default=False, help = 'Indicates if we want to store a report on outputfile+"_.report".')
    group.add_argument('--factor_addition', action='store_true', dest='factoraddition', default=False, help = 'Indicates we want to append factors to language.')
    group.add_argument('--factor_adaptation', '--fa',  action='store_true', dest='factoradaptation', default=False, help = 'Indicates we want to factor_adapt language features.')
    group.add_argument('--res_factor_adaptation', '--rfa' , action='store_true', dest='residualizedfactoradaptation', default=False, help = 'Indicates we want to apply residualized factor adaptation.')


    group = parser.add_argument_group('Classification Actions', '')
    group.add_argument('--train_classifiers', '--train_class', action='store_true', dest='trainclassifiers', default=False,
                       help='train classification models for each outcome field based on feature table')
    group.add_argument('--test_classifiers', action='store_true', dest='testclassifiers', default=False,
                       help='trains and tests classification for each outcome')
    group.add_argument('--nfold_classifiers', '--combo_test_classifiers', '--nfold_test_classifiers', action='store_true', dest='combotestclassifiers', default=False,
                       help='train/test a regression model with and without all combinations of controls')
    group.add_argument('--predict_classifiers', '--predict_class', action='store_true', dest='predictclassifiers', default=False,
                       help='predict outcomes bases on loaded training')
    group.add_argument('--roc', action='store_true', dest='roc',
                       help="Computes ROC curves and outputs to PDF")
    group.add_argument('--predict_classifiers_to_feats', type=str, dest='predictctofeats', default=None,
                       help='predict outcomes into a feature file (provide a name)')
    group.add_argument('--predict_probabilities_to_feats', '--predict_probs_to_feats', type=str, dest='predictprobstofeats', default=None,
                       help='predict probabilities into a feature file (provide a name)')
    group.add_argument('--predict_classifiers_to_outcome_table', type=str, dest='predictCtoOutcomeTable', default=None,
                       help='predict outcomes into an outcome table (provide a name)')
    group.add_argument('--regression_to_lexicon', dest='regrToLex', type=str, default=None,
                       help='Uses the regression coefficients to create a weighted lexicon.')
    group.add_argument('--classification_to_lexicon', dest='classToLex', type=str, default=None,
                       help='Uses the classification coefficients to create a weighted lexicon.')
    group.add_argument('--stratify_folds', action='store_true', dest='stratifyfolds', default=False,
                       help='stratify folds during combo_test_classifiers or combo_test_regression')
    group.add_argument('--train_c2r', action='store_true', dest='trainclasstoreg', default=False,
                       help='train a model that goes from classification to prediction')
    group.add_argument('--test_c2r', action='store_true', dest='testclasstoreg', default=False,
                       help='trains and tests classification for each outcome')
    group.add_argument('--predict_c2r', action='store_true', dest='predictclasstoreg', default=False,
                       help='predict w/ classification to regression model')

    group = parser.add_argument_group('Clustering Actions', '')
    group.add_argument('--reducer_to_lexicon', type=str, dest='reducertolexicon', default=None,
                       help='writes the reduction model to a specified lexicon')
    group.add_argument('--super_topics', type=str, dest='supertopics', default=None,
                       help='unroll reduced topics to the word level')
    group.add_argument('--reduced_lexicon', '--reduced_lex',  type=str, dest='reducedlexicon', default=None,
                       help='use during super topics creation if you have already run --reducer_to_lexicon')
    group.add_argument('--fit_reducer', action='store_true', dest='fitreducer', default=False,
                       help='reduces a feature space to clusters')
    group.add_argument('--num_factors', '--n_components', '-k', dest='n_components', default=None,
                       help='Number of factors in clustering method. Used with --fit_reducer.')
    group.add_argument('--transform_to_feats', dest='transformdrtofeats', type=str, default = None,
                       help='feature table to send reduced features to.')
    
    group = parser.add_argument_group('CCA Actions', '')
    group.add_argument('--cca', type=int, dest='cca', default=0,
                       help='Performs sparse CCA on a set of features and a set of outcomes.'+
                       "Argument is number of components to output (Uses R's PMA package)")
    group.add_argument('--cca_penalty_feats', '--cca_penaltyx', type=float, dest='penaltyFeats', default = None,
                       help="Penalty value on the feature matrix (X) [penaltyx argument of PMA.CCA]"+
                       "must be between 0 and 1, larger means less penalization (i.e. less sparse) ")
    group.add_argument('--cca_penalty_outcomes', '--cca_penaltyz', type=float, dest='penaltyOutcomes', default = None,
                       help="Penalty value on the outcomes matrix (Z) [penaltyz argument of PMA.CCA]"+
                       "must be between 0 and 1, larger means less penalization (i.e. less sparse) ")
    group.add_argument('--cca_outcomes_vs_controls', dest='ccaOutcomesVsControls',action='store_true',
                       help="performs CCA on outcomes vs controls (no language)")
    group.add_argument('--cca_permute', dest='ccaPermute', type=int,default=0,
                       help='Wrapper for the PMA package CCA.permute function that determines the'+
                       ' ideal L1 Penalties for X and Z matrices'+
                       'argument: number of permutations')
    group.add_argument('--cca_predict_components', dest='predictCcaCompsFromModel',action="store_true",
                       help='Using --picklefile, predict outcomes from the V matrix (aka Z_comp)')
    group.add_argument('--to_sql_table', dest='newSQLtable',type=str,
                       help='Using --cca_predict_components, predict components to sql table,'+
                       'the name of which you should provide here')
    group.add_argument('--useXFeats', dest='usexfeats',action="store_true", default = False,
                       help='Use feats stored in X matrix when predicting CCA components to SQL')
    group.add_argument('--useXControls', dest='usexcontrols',action="store_true", default = False,
                       help='Use controls stored in X matrix when predicting CCA components to SQL')

    group.add_argument('--save_models', action='store_true', dest='savemodels', default=False,
                       help='saves predictive models (uses --picklefile)')
    group.add_argument('--load_models', action='store_true', dest='loadmodels', default=False,
                       help='loads predictive models (uses --picklefile)')


    group = parser.add_argument_group('Plot Actions', '')
    group.add_argument('--barplot', action='store_true', dest='barplot',
                       help='produce correlation barplots. Requires fg, oa. Uses groupfreqthresh, outputdir')
    group.add_argument('--scatterplot', action='store_true', dest='scatterplot',
                       help='Requires --outcome_table --outcome_fields, optional: -f --feature_names')
    group.add_argument('--feat_flexibin', action='store_true', dest='featflexibin', default=False,
                       help='Plots a binned feature table, uses --num_bins, --group_id_range, --feat_table, --flexiplot_file')    # group.add_argument('--hist2d', action='store_true', dest='hist2d',
    group.add_argument('--skip_bin_step', action='store_true', dest='skipbinstep', default=False,
                       help='Skips the binning step for feat_flexibin. For when we want fast plotting and the flexitable has been created.')
    group.add_argument('--preserve_bin_table', action='store_true', dest='preservebintable', default=False,
                       help='Preserves the flexibin table for faster plotting.')
    # group.add_argument('--hist
    # group.add_argument('--hist2d', action='store_true', dest='hist2d',
    #                    help='Requires -f --feature_names --outcome_table --outcome_value')
    group.add_argument('--descplot', action='store_true', dest='descplot',
                       help='produce histograms and boxplots for specified outcomes. Requires oa. Uses outputdir')
    group.add_argument('--loessplot', type=str, metavar='FEAT(S)', dest='loessplot', nargs='+', default='',
                       help='Output loess plots of the given features.')

    group = parser.add_argument_group('LDA Estimation Actions', '')
    group.add_argument('--estimate_lda_topics', action='store_true', help="Estimates LDA topics using PyMallet.")
    group.add_argument('--mallet_path', help="Specify the path to the Mallet executable. If unspecified, "
                                             "LDA estimation is performed with PyMallet.")
    group.add_argument('--save_lda_files', help="The directory in which to save LDA estimation files. Default: "
                                                "current working directory.")
    group.add_argument('--lda_lexicon_name', help="The name of the LDA topic-lexicon. Required unless --no_lda_lexicon "
                                                  "is used.")
    group.add_argument('--no_lda_lexicon', action='store_true', help="Do not store the LDA topics as a lexicon.")
    group.add_argument('--num_topics', type=int, default=dlac.DEF_NUM_TOPICS,
                       help="The number of LDA topics to estimate. Default: {}".format(str(dlac.DEF_NUM_TOPICS)))
    group.add_argument('--num_lda_threads', type=int, default=dlac.DEF_NUM_THREADS,
                       help="The number of parallel threads to use by Mallet. Does not affect PyMallet. Default: {"
                            "}".format(str(dlac.DEF_NUM_THREADS)))
    group.add_argument('--num_stopwords', type=int, default=dlac.DEF_NUM_STOPWORDS, help="The number of stopwords to "
                                                                                         "use.")
    group.add_argument('--extra_lda_stopwords', help="Supply a file containing extra stopwords to use in estimating "
                                                     "LDA topics. One stopword per line.")
    group.add_argument('--no_lda_stopping', action='store_true', help="Disables stopping")
    group.add_argument('--lda_alpha', type=float, default=dlac.DEF_ALPHA,
                       help="The LDA alpha parameter. Default: {}".format(str(dlac.DEF_ALPHA)))
    group.add_argument('--lda_beta', type=float, default=dlac.DEF_BETA,
                       help="The LDA beta parameter. Currently ignored by Mallet. Default: {}".format(str(
                           dlac.DEF_BETA)))
    group.add_argument('--lda_iterations', type=int, default=dlac.DEF_NUM_ITERATIONS,
                       help="The number of Gibbs sampling iterations to perform. Default: {}"
                       .format(str(dlac.DEF_NUM_ITERATIONS)))

    if len(sys.argv)==1:
        parser.print_help()
        sys.exit(0)

    if fn_args:
        args = parser.parse_args(fn_args.split())
    else:
        args = parser.parse_args(remaining_argv)

    ##Warnings
    if not args.bonferroni:
      print("--no_bonf has been depricated. Default p correction method is now Benjamini, Hochberg. Please use --no_correction instead of --no_bonf.")
      sys.exit(1)

    ##Argument adjustments:
    if not args.valuefunc: args.valuefunc = lambda d: d
    if not args.lexvaluefunc: args.lexvaluefunc = lambda d: d

    if args.feattable:
        if isinstance(args.feattable,list):
            args.feattable = [s.strip() for s in args.feattable]
        else:
            args.feattable = arg.feattable.strip()


    if args.outcomeWithOutcomeOnly and not args.feattable:
        args.groupfreqthresh = 0


    if args.feattable and len(args.feattable) == 1:
        args.feattable = args.feattable[0]

    if not args.feattable and args.aggregategroup:
        args.feattable = aggregategroup[0]

    if args.weightedeval:
        args.outcomefields.append(args.weightedeval)

    if args.weightedsample:
        args.outcomefields.append(args.weightedsample)

    if args.makewordclouds:
        if not args.tagcloud:
            print("WARNING: --make_wordclouds used without --tagcloud, setting --tagcloud to True")
            args.tagcloud = True

    if args.maketopicwordclouds:
        if not args.topictc and not args.corptopictc:
            print("WARNING: --make_topic_wordcloud used without --topic_tagcloud or --corp_topic_tagcloud, setting --topic_tagcloud to True")
            args.topictc = True

    if not args.encoding:
        if not args.useunicode:
            args.encoding = 'latin1'
        else:
            args.encoding = dlac.DEF_ENCODING

    if not args.groupfreqthresh and args.groupfreqthresh != 0:
        setGFTWarning = False
        args.groupfreqthresh = dlac.getGroupFreqThresh(args.correl_field)
    else:
        setGFTWarning = True

    args.integrationmethod = ''
    if args.factoradaptation:
        args.integrationmethod = 'fa'
        if args.factoraddition:
            args.integrationmethod += '_plus'
    elif args.residualizedfactoradaptation:
        args.integrationmethod = 'rfa'
        if args.factoraddition:
            args.integrationmethod += '_plus'
    elif args.factoraddition:
        args.integrationmethod = 'plus'

    if args.adaptationfactors:
        args.outcomefields = args.outcomefields + args.adaptationfactors
    elif args.factoradaptation or args.residualizedfactoradaptation or args.factoraddition:
        args.adaptationfactors = args.outcomecontrols
        args.outcomefields = args.outcomefields + args.adaptationfactors

    if args.fsparams:
        args.featureselectionparams = { 'kbest': args.kbest , 'pca': args.pcacomp }
    else:
        args.featureselectionparams = None

    DLAWorker.lexicon_db = args.lexicondb

    ##Process Arguments
    def DLAW():
        return DLAWorker(args.dbengine, args.corpdb, args.corptable, args.correl_field, args.mysqlconfigfile, args.message_field, args.messageid_field, args.encoding, args.useunicode, args.lexicondb, wordTable = args.wordTable)

    def MA():
        return MessageAnnotator(args.dbengine, args.corpdb, args.corptable, args.correl_field, args.mysqlconfigfile, args.message_field, args.messageid_field, args.encoding, args.useunicode, args.lexicondb, wordTable = args.wordTable)

    def MT():
        return MessageTransformer(args.dbengine, args.corpdb, args.corptable, args.correl_field, args.mysqlconfigfile, args.message_field, args.messageid_field, args.encoding, args.useunicode, args.lexicondb, wordTable = args.wordTable)

    def FE():
        return FeatureExtractor(args.dbengine, args.corpdb, args.corptable, args.correl_field, args.mysqlconfigfile, args.message_field, args.messageid_field, args.encoding, args.useunicode, args.lexicondb, wordTable = args.wordTable)

    def SE():
        return SemanticsExtractor(args.dbengine, args.corpdb, args.corptable, args.correl_field, args.mysqlconfigfile, args.message_field, args.messageid_field, args.encoding, args.useunicode, args.lexicondb, args.corpdir, wordTable = args.wordTable)

    def OG():
        return OutcomeGetter(args.dbengine, args.corpdb, args.corptable, args.correl_field, args.mysqlconfigfile, args.message_field, args.messageid_field, args.encoding, args.useunicode, args.lexicondb, args.outcometable, args.outcomefields, args.outcomecontrols, args.outcomeinteraction, args.cattobinfields, args.cattointfields, args.groupfreqthresh, args.low_variance_thresh, args.featlabelmaptable, args.featlabelmaplex, wordTable = args.wordTable, fold_column = args.fold_column)

    def OA():
        return OutcomeAnalyzer(args.dbengine, args.corpdb, args.corptable, args.correl_field, args.mysqlconfigfile, args.message_field, args.messageid_field, args.encoding, args.useunicode, args.lexicondb, args.outcometable, args.outcomefields, args.outcomecontrols, args.outcomeinteraction, args.cattobinfields, args.cattointfields, args.groupfreqthresh, args.low_variance_thresh, args.featlabelmaptable, args.featlabelmaplex, wordTable = args.wordTable, output_name = args.outputname)

    def FR():
        return FeatureRefiner(args.dbengine, args.corpdb, args.corptable, args.correl_field, args.mysqlconfigfile, args.message_field, args.messageid_field, args.encoding, args.useunicode, args.lexicondb, args.feattable, args.featnames, wordTable = args.wordTable)

    def FG(featTable = None):
        if not featTable:
            featTable = args.feattable
        return FeatureGetter(args.dbengine, args.corpdb, args.corptable, args.correl_field, args.mysqlconfigfile, args.message_field, args.messageid_field, args.encoding, args.useunicode, args.lexicondb, featTable, args.featnames, wordTable = args.wordTable)

    def FGs(featTable = None):
        if not featTable:
            featTable = args.feattable
            if not featTable:
                print("Need to specify feature table(s)")
                sys.exit(1)
        if isinstance(featTable, str):
            featTable = [featTable]
        return [FeatureGetter(args.dbengine, 
                              args.corpdb,
                              args.corptable,
                              args.correl_field,
                              args.mysqlconfigfile, 
                              args.message_field,
                              args.messageid_field,
                              args.encoding,
                              args.useunicode,
                              args.lexicondb, featTable,
                              args.featnames,
                              wordTable = args.wordTable)
                for featTable in featTable]
    def TE():
        return TopicExtractor(args.dbengine, args.corpdb, args.corptable, args.correl_field, args.mysqlconfigfile, args.message_field, args.messageid_field, dlac.DEF_ENCODING, dlac.DEF_UNICODE_SWITCH, args.ldamsgtbl)

    dlaw = None
    ma = None
    mt = None
    fe = None
    se = None
    fr = None
    og = None
    oa = None
    fg = None
    fgs = None #feature getters
    te = None

    # if not fe:
    #  fe = FE()
    # fe.addFeatsToLexTable(args.lextable, valueFunc = args.valuefunc, isWeighted=args.weightedlexicon, featValueFunc=args.lexvaluefunc)
    # exit()

    # SQL interface methods
    if args.listfeattables or args.showtables or args.describetables or args.createrandsample or args.viewtables or args.createcopiedtable:
        if not dlaw: dlaw = DLAW()

    if isinstance(args.describetables, list) and len(args.describetables) == 0:
        if not dlaw: dlaw = DLAW()
        args.describetables = True

    if isinstance(args.viewtables, list) and len(args.viewtables) == 0:
        if not dlaw: dlaw = DLAW()
        args.viewtables = True

    if args.listfeattables or args.showtables:
        feat_table = True if args.listfeattables else False
        tables = dlaw.getTables(like=args.showtables, feat_table=feat_table)
        print('\nFound %s available tables\n----' % (len(tables)))
        for table in tables: print(str(table[0]))

    def printTableDesc(description):
        header = ['Field', 'Type','Null', 'Key', 'Default', 'Extra']
        row_format ="{:>25}{:>25}{:>10}{:>10}{:>10}{:>15}"
        print(row_format.format(*header))
        for row in description:
            print(row_format.format(*[r if r or r == 0 else '' for r in row]))

    def printTableData(data):
        row_format = "{:>15}" * len(data[0])
        for row in data:
            print(row_format.format(*[' ' + str(r)[0:14] if r or r == 0 else '' for r in row]))

    if args.describetables:
        if args.corptable:
            printTableDesc(dlaw.describeTable(table_name=dlaw.corptable))
        if isinstance(args.feattable, str):
            printTableDesc(dlaw.describeTable(table_name=args.feattable))
        elif isinstance(args.feattable, list):
            for ftable in args.feattable:
                printTableDesc(dlaw.describeTable(table_name=ftable))
        if args.outcometable:
            printTableDesc(dlaw.describeTable(table_name=args.outcometable))
        if isinstance(args.describetables, str): args.describetables = [args.describetables]
        if isinstance(args.describetables, list):
            for tbl in args.describetables: printTableDesc(dlaw.describeTable(table_name=tbl))

    if args.viewtables:
        if args.corptable:
            printTableData(dlaw.viewTable(table_name=dlaw.corptable))
        if isinstance(args.feattable, str):
            printTableData(dlaw.viewTable(table_name=args.feattable))
        elif isinstance(args.feattable, list):
            for ftable in args.feattable:
                printTableData(dlaw.viewTable(table_name=ftable))
        if args.outcometable:
            printTableData(dlaw.viewTable(table_name=args.outcometable))
        if isinstance(args.viewtables, str): args.viewtables = [args.viewtables]
        if isinstance(args.viewtables, list):
            for tbl in args.viewtables: printTableData(dlaw.viewTable(table_name=tbl))

    if args.createrandsample:
        if len(args.createrandsample) > 2:
            print("Error: Only two optional arguments for --create_random_sample")
            sys.exit(1)
        percentage, random_seed = args.createrandsample if len(args.createrandsample) > 1 else (args.createrandsample[0], dlac.DEFAULT_RANDOM_SEED)
        rand_table = dlaw.createRandomSample(float(percentage), random_seed, where=args.groupswhere)

    if args.createcopiedtable:
        if len(args.createcopiedtable) != 2:
            print("Error: Need two arguments for --create_copied_table")
            sys.exit(1)
        new_table = dlaw.createCopiedTable(args.createcopiedtable[0], args.createcopiedtable[1], where=args.groupswhere)
    
    #Feature Extraction:
    if args.addngrams:
        if not fe: fe = FE()

        if args.use_collocs:
            pmi_filter_thresh = args.colloc_pmi_thresh if args.colloc_pmi_thresh else dlac.DEF_PMI
            collocs_list = fe._getCollocsFromTable(args.colloc_table, pmi_filter_thresh, args.colloc_column, dlac.DEF_COLUMN_PMI_FILTER)
            if args.feature_type_name:
                feature_type_name = args.feature_type_name
            else:
#                feature_type_name = "clc" + str(pmi_filter_thresh).replace('.', '_')
                feature_type_name = "colloc"
            args.feattable = fe.addCollocFeatTable(collocs_list, lowercase_only=args.lowercaseonly, valueFunc = args.valuefunc, includeSubCollocs=args.include_sub_collocs, featureTypeName = feature_type_name)

        elif args.gzipcsv:
            args.feattable = fe.addNGramTableGzipCsv(args.n, args.gzipcsv, 3, 0, 19, lowercase_only=args.lowercaseonly, valueFunc = args.valuefunc)

        else:
            ftables = list()
            for n in args.n:
                ftables.append(fe.addNGramTable(n, lowercase_only=args.lowercaseonly, valueFunc = args.valuefunc, metaFeatures = args.metafeats, extension = args.extension))
            if len(ftables) > 1:
                args.feattable = ftables;
            else:
                args.feattable = ftables[0]
    if args.addngramsfromtok:
        if not fe: fe = FE()
        ftables = list()
        for n in args.n:
            ftables.append(fe.addNGramTableFromTok(n, lowercase_only=args.lowercaseonly, valueFunc = args.valuefunc, metaFeatures = args.metafeats))
            if len(ftables) > 1:
                args.feattable = ftables;
            else:
                args.feattable = ftables[0]

    if args.addcharngrams:
        if not fe: fe = FE()

        #elif args.gzipcsv:
        #    args.feattable = fe.addNGramTableGzipCsv(args.n, args.gzipcsv, 3, 0, 19, lowercase_only=args.lowercaseonly, valueFunc = args.valuefunc)

        ftables = list()
        for n in args.n:
            ftables.append(fe.addCharNGramTable(n, lowercase_only=args.lowercaseonly, valueFunc = args.valuefunc, metaFeatures = args.metafeats))
        if len(ftables) > 1:
            args.feattable = ftables;
        else:
            args.feattable = ftables[0]

    if args.addlextable:
        if not fe: fe = FE()
        if args.multicategory_normalization and not args.lexicon_normalization:
            args.lexicon_normalization = True
        args.feattable = fe.addLexiconFeat(args.lextable, lowercase_only=args.lowercaseonly,
                                           valueFunc=args.valuefunc, isWeighted=args.weightedlexicon,
                                           featValueFunc=args.lexvaluefunc, extension=args.extension,
                                           lexicon_weighting=args.lexicon_normalization, multicategory_weighting=args.multicategory_normalization)

    if args.addcorplextable:
        if not args.lextable:
            print("Need to specify lex table with -l", file=sys.stderr)
            sys.exit()
        if not fe: fe = FE()
        args.feattable = fe.addCorpLexTable(args.lextable, lowercase_only=args.lowercaseonly, valueFunc = args.valuefunc, isWeighted=args.weightedlexicon, featValueFunc=args.lexvaluefunc)

    if args.addphrasetable:
        if not fe: fe = FE()
        args.feattable = fe.addPhraseTable(valueFunc = args.valuefunc)

    if args.addpostable or args.pos_ngram:
        if not fe: fe = FE()
        args.feattable = fe.addPosTable(valueFunc = args.valuefunc, keep_words = args.pos_ngram)

    if args.embaddfeat:
        if not fe: fe = FE()
        args.feattable = fe.addEmbTable(modelName = args.embmodel, tokenizerName=args.tokenizermodel, modelClass = args.embclass, batchSize=args.batchsize, aggregations=args.embaggs, layersToKeep=args.emblayers, noContext=args.embnocontext, layerAggregations = args.emblayeraggs, wordAggregations=args.transwordaggs, valueFunc = args.valuefunc, customTableName = args.embtablename, keepMsgFeats = args.embkeepmsg)
    
    if args.addldafeattable:
        if not fe: fe = FE()
        args.feattable = fe.addLDAFeatTable(args.addldafeattable, valueFunc = args.valuefunc)

    if args.addpnames:
        if not fe: fe = FE()
        namesLex = lexInterface.Lexicon(mysql_config_file=args.mysqlconfigfile)
        namesLex.loadLexicon(args.addpnames[0])
        langLex = lexInterface.Lexicon(mysql_config_file=args.mysqlconfigfile)
        langLex.loadLexicon(args.addpnames[1])
        args.feattable = fe.addPNamesTable(namesLex.getLexicon(), langLex.getLexicon(),  valueFunc = args.valuefunc)

    if args.addwnnopos:
        if not fe: fe = FE()
        args.feattable = fe.addWNNoPosFeat(valueFunc = args.valuefunc, featValueFunc=args.lexvaluefunc)

    if args.addwnpos:
        if not fe: fe = FE()
        args.feattable = fe.addWNPosFeat(pos_table = args.feattable, valueFunc = args.valuefunc, featValueFunc=args.lexvaluefunc)

    if args.addfkscore:
        if not fe: fe = FE()
        args.feattable = fe.addFleschKincaidTable(valueFunc = args.valuefunc)

    if args.estimate_lda_topics:
        if not args.feattable:
            raise Exception('LDA topic estimation requires a feature table.')
        if not args.lda_lexicon_name and not args.no_lda_lexicon:
            raise Exception('Must specify an LDA lexicon name with --lda_lexicon_name or disable topic-lexicon with '
                            '--no_lda_lexicon.')
        if args.save_lda_files is None:
            args.save_lda_files = os.getcwd()
        if not os.path.isdir(args.save_lda_files):
            try:
                os.makedirs(args.save_lda_files)
            except FileExistsError:
                raise Exception('--save_lda_files path is already a file: {}'.format(args.save_lda_files))
            except PermissionError:
                raise Exception('You do not have permission to create a directory at {}'.format(args.save_lda_files))
        if not args.printjoinedfeaturelines:  # allow users to specify a file if they want to write it out
            args.printjoinedfeaturelines = os.path.join(args.save_lda_files, 'lda_msgs.txt')
        args.createdists = True

    if args.printjoinedfeaturelines:
        print('Writing joined feature lines file: {}.'.format(args.printjoinedfeaturelines))
        if not fg: fg = FG()
        fg.printJoinedFeatureLines(args.printjoinedfeaturelines)

    if args.estimate_lda_topics:
        print('Estimating LDA topics. This may take a long time!')
        if not fg: fg = FG()
        lda_estimator = LDAEstimator(fg, args.num_topics, args.lda_alpha, args.lda_beta,
                                     args.lda_iterations, args.num_stopwords, args.no_lda_stopping,
                                     files_dir=args.save_lda_files, num_threads=args.num_lda_threads,
                                     extra_stopwords_file=args.extra_lda_stopwords, language=args.language)
        state_file = lda_estimator.estimate_topics(args.printjoinedfeaturelines, args.mallet_path)
        if args.mallet_path:  # only need to add message IDs if using mallet; pymallet handles this for us
            args.addmessageid = [args.printjoinedfeaturelines, state_file]
            state_file = os.path.splitext(state_file)[0]
            args.outputname = state_file
        args.addldamsgs = state_file

    # LDA
    if args.addmessageid:
        messageFile = open(args.addmessageid[0], 'rb')
        stateFile = gzip.open(args.addmessageid[1], 'rb')
        writeFile = open(args.outputname, 'w')
        writeFile.write(stateFile.readline().decode())
        writeFile.write(stateFile.readline().decode())
        writeFile.write(stateFile.readline().decode())
        error = sys.stderr #open('error.log.txt', 'w+')

        currentindex = -1
        messageid = -1
        while (True):
            line = stateFile.readline().decode()
            if len(line) == 0:
                break
            tokens = line.split()
            if tokens[0] != currentindex:
                currentindex = tokens[0]
                line_read = messageFile.readline()
                if len(line_read.split()) > 0:
                    messageid = line_read.split()[0]
                    if str(messageid) == '$':
                        error.write("Error line: " + line_read)

            tokens[1] = messageid.decode()
            line = ' '.join(tokens)
            line += '\n'
            writeFile.write(line)
        messageFile.close()
        stateFile.close()
        writeFile.close()

    # transform message tables
    if args.addldamsgs and not args.no_lda_lexicon:
        lda_state_name = None
        if args.estimate_lda_topics:
            lda_state_name = args.lda_lexicon_name.replace('-', '_') if args.lda_lexicon_name else 'lda_topics'
        if not mt:
            mt = MT()
        print('Importing LDA state.')
        mt.addLDAMessages(args.addldamsgs, ldaStatesName=lda_state_name)

    if args.addtokenized:
        if not mt: mt = MT()
        mt.addTokenizedMessages()

    if args.addsegmented:
        if not mt: mt = MT()
        mt.addSegmentedMessages(args.segmentationModel)

    if args.addtweetpos:
        if not mt: mt = MT()
        mt.addTweetPOSMessages()

    if args.addtweettok:
        if not mt: mt = MT()
        mt.addTweetTokenizedMessages()

    if args.addsenttokenized or args.addsentperrow:
        if not mt: mt = MT()
        mt.addSentTokenizedMessages(args.addsentperrow, args.cleanmessages)

    if args.printtokenizedlines:
        if not mt: mt = MT()
        if args.feat_whitelist:
            mt.printTokenizedLines(args.printtokenizedlines, whiteListFeatTable = args.feat_whitelist)
            #TODO: change whitelistfeat table to get a list instead.
        else:
            mt.printTokenizedLines(args.printtokenizedlines)

    if args.addparses:
        if not mt: mt = MT()
        mt.addParsedMessages()

    # annotate message tables
    if args.langfilter or args.lightenglishfilter:
        if not ma: ma = MA()
        ma.addLanguageFilterTable(args.langfilter, args.cleanmessages, args.lowercaseonly, args.lightenglishfilter)

    if args.deduplicate:
        if not ma: ma = MA()
        ma.addDedupFilterTable(anonymize=args.cleanmessages)

    if args.spamfilter:
        if not ma: ma = MA()
        ma.addSpamFilterTable(threshold=args.spamfilter)

    if args.cleanmessages and not (args.langfilter or args.deduplicate) :
        if not ma: ma = MA()
        ma.addAnonymizedTable()

    #LDA
    if args.createdists and not args.no_lda_lexicon:
        dist_file_output_name = None
        if args.estimate_lda_topics:
            dist_file_output_name = os.path.join(args.save_lda_files, 'lda')
            args.ldamsgtbl = '{}_lda${}'.format(args.corptable, lda_state_name)
            te = TopicExtractor(args.dbengine, args.corpdb, args.corptable, args.correl_field, args.mysqlconfigfile, args.message_field,
                                args.messageid_field, dlac.DEF_ENCODING, dlac.DEF_UNICODE_SWITCH, args.ldamsgtbl)
        elif not te:
            te = TE()
        te.createDistributions(dist_file_output_name)

    #LDA
    if args.estimate_lda_topics:
        if not args.no_lda_lexicon:
            
            if not dlaw: dlaw = DLAW()

            topic_file = os.path.join(args.save_lda_files, 'lda.topicGivenWord.csv')
            create_name = '{}_cp'.format(args.lda_lexicon_name)
            print('Adding topic lexicon: {}'.format(create_name))
            dlaw.load_lexicon(topic_file, "topicCSV", create_name)

            topic_file = os.path.join(args.save_lda_files, 'lda.freq.threshed50.loglik.csv')
            create_name = '{}_freq_t50ll'.format(args.lda_lexicon_name)
            print('Adding topic lexicon: {}'.format(create_name))
            dlaw.load_lexicon(topic_file, "topicCSV", create_name)

        print('LDA topics estimated. Files saved in {}.'.format(args.save_lda_files))

    if args.addtopiclexfromtopicfile:
        if not fe: fe = FE()
        if not args.topicfile or not args.topiclexicon or not args.topiclexmethod:
            raise Exception('cannot add topic lexicon from topic file without specifying --topic_file, --topic_lexicon, --topic_lex_method')
#--topic_file --topic_lexicon --lex_thresh --topic_method')
        if args.lexthresh == None: args.lexthresh = float('-inf')
        fe.addTopicLexFromTopicFile(args.topicfile, args.topiclexicon, args.topiclexmethod, args.lexthresh)

    if args.addoutcomefeats:
        if not fe: fe = FE()
        if not og: og = OG()
        fe.addOutcomeFeatTable(og, valueFunc = args.valuefunc)

    if args.addtimexdiff:
        if not fe: fe = FE()
        args.feattable = fe.addTimexDiffFeatTable(args.date_field)

    if args.addpostimexdiff:
        if not fe: fe = FE()
        args.feattable = fe.addPOSAndTimexDiffFeatTable(args.date_field, valueFunc = args.valuefunc)

    #Semantic Feature Extraction:
    if args.addner:
        if not se: se = SE()
        args.feattable = se.addNERTable(valueFunc = args.valuefunc)




    #Feature Refinement:

    #(first do refinements that group tables so these tables can then be refined further by filter refinements)
    if args.combinefeattables:
        if not fr: fr=FR()
        args.feattable = fr.createCombinedFeatureTable(args.combinefeattables, args.feattable)
        #TODO: use internal fr variable for feature tables rather than argument
        fr = None #so that the feature table must be re-taken

    if args.featgroupoutcomes:
        if not fr: fr=FR()
        if not og: og=OG()
        args.feattable = fr.createFeatTableByDistinctOutcomes(og, nameSuffix=args.outputname)

    if args.aggregategroup:
        if not fr: fr=FR()
        args.feattable = fr.createAggregateFeatTableByGroup(valueFunc = args.valuefunc)

    if args.interpolategroup:
        if not fr: fr=FR()
        args.feattable = fr.createInterpolatedFeatTable(days = args.interpolategroup, dateField = args.date_field, groupFreqThresh = args.groupfreqthresh, where = args.groupswhere)


    if args.featoccfilter:
        if args.use_collocs and not args.wordTable:
            args.wordTable = args.feattable
        if not fr: fr=FR()
        args.feattable = fr.createTableWithRemovedFeats(args.pocc, args.minfeatsum, args.groupfreqthresh, setGFTWarning)

    if args.featcollocfilter:
        if not fr: fr=FR()
        if args.use_collocs:
            raise NotImplementedError
        # args.feattable = fr.createCollocRefinedFeatTable(featNormTable=False) #faster ## FEAT NORM TABLE LUKASZ LUKE
        args.feattable = fr.createCollocRefinedFeatTable(args.pmi)

    if args.archetypesimilarity:
        if not fr: fr=FR()
        args.feattable = fr.createArchetypeFeats(args.archetypesimilarity)

        
    if args.maketopiclabelmap:
        if not fr: fr=FR()
        args.featlabelmaplex = fr.makeTopicLabelMap(args.topiclexicon, args.numtopicwords, args.weightedlexicon)

    if args.featflexibin:
        if not fr: fr=FR()
        if not args.numbins or not args.groupidrange or not args.feattable:
            raise Exception('cannot bin feature table without specifying --num_bins, --group_id_range, --feat_table') #optionally specify --topic_lexicon
        args.feattable = fr.createTableWithBinnedFeats(args.numbins, args.groupidrange, args.valuefunc, args.gender, args.genderattack, skip_binning=args.skipbinstep)#, args.reportingpercent)
        # sys.exit(0)
        # need feature csv for batch plotting
        temp_feature_file = OutcomeGetter.buildBatchPlotFile(args.corpdb, args.feattable, args.topiclist) if not args.flexiplotfile else args.flexiplotfile
        feat_to_label = None
        if args.topiclexicon:
            if not oa: oa=OA()
            feat_to_label = oa.buildTopicLabelDict(args.topiclexicon, args.numtopicwords) #TODO-finish -- uses topictagcloudwords method
            #pprint(feat_to_label)
        OutcomeGetter.plotFlexibinnedTable(args.corpdb, args.feattable, temp_feature_file, feat_to_label, args.preservebintable)
    if args.tfidf:
        if not fr: fr=FR()
        args.feattable = fr.createTfIdfTable(args.feattable)

    if args.createcollocscores:
        if not fr: fr=FR()
        fr.creatCollocScores(args.createcollocscores)

    #if args.addmean: #works, but excessive option
    #    if not fr: fr=FR()
    #    fr.addFeatTableMeans()

    if args.addfeatnorms:
        if not fr: fr=FR()
        fr.addFeatNorms(groupFreqThresh = args.groupfreqthresh, setGFTWarning = setGFTWarning)

    if args.addstdfeats:
        if not fr: fr=FR()
        fr.createStandardizedFeatTable(groupFreqThresh = args.groupfreqthresh, setGFTWarning = setGFTWarning)

    #create whitelist / blacklist
    if args.categories:
        if isinstance(args.categories, str):
            args.categories = [args.categories]
    if args.feat_blacklist:
        if isinstance(args.feat_blacklist, str):
            args.feat_blacklist = [args.feat_blacklist]
    if args.feat_whitelist:
        if isinstance(args.feat_whitelist, str):
            args.feat_whitelist = [args.feat_whitelist]
    (whitelist, blacklist) = (None, None)
    # Wildcards are not handled!!!
    if args.blacklist:
        if not dlaw: dlaw = DLAW()
        blacklist = dlaw.makeBlackWhiteList(args.feat_blacklist, args.lextable, args.categories)
    if args.whitelist:
        if not dlaw: dlaw = DLAW()
        whitelist = dlaw.makeBlackWhiteList(args.feat_whitelist, args.lextable, args.categories)

    def makeOutputFilename(args, fg=None, og=None, prefix=None, suffix=None):
        if args.outputname:
            return args.outputname+suffix

        outputFile = args.outputdir + '/'
        filePieces = []
        if prefix:
            filePieces.append(prefix)
        if fg:
            filePieces.append(fg.featureTable)
        if og:
            if og.outcome_value_fields and len(og.outcome_value_fields) <= 10:
                filePieces.append('_'.join(og.outcome_value_fields))
        if og.outcome_controls: filePieces.append('_'.join(og.outcome_controls))
        if args.groupfreqthresh: filePieces.append("freq" + str(args.groupfreqthresh))
        if args.spearman: filePieces.append('spear')
        if args.blacklist: filePieces.append('blist')
        if args.lextable: filePieces.append('-'.join([args.lextable] + args.categories))
        if args.feat_blacklist: filePieces.append('-'.join(args.feat_blacklist))
        if args.feat_whitelist: filePieces.append('wlist')
        if args.feat_whitelist: filePieces.append('-'.join(args.feat_whitelist))
        if suffix: filePieces.append(suffix)
        outputFile += '.'.join(filePieces)
        outputFile = outputFile.replace("$", ".")
        print("Created output filename: %s" % outputFile)
        return outputFile

    #Feature Only options:
    if args.top_messages:
      if not args.feattable:
        print("You must specify a feature table, extracted at the message level.")
        sys.exit()
      if not any(substring in args.feattable for substring in ["mess", "msg"]):
        print("WARNING: Your feature table doesn't seem to be at the message level.")
      if not fg: fg = FG()

      outputFile = makeOutputFilename(args, None, None, suffix="_topmsgs.csv") if args.outputname else None
      fg.getTopMessages(args.lextable, outputFile, args.top_messages, args.feat_whitelist, group_freq_thresh=args.groupfreqthresh)

    #Outcome Only options:
    if args.p_correction_method == '':
        args.bonferroni = False

    if args.printcsv:
        pprint(args)
        if not oa: oa = OA()
        if not fg: fg = FG()
#        if args.isbinned:
#            og.printBinnedGroupsAndOutcomesToCSV(fg, args.printcsv)
#        else:
        oa.printGroupsAndOutcomesToCSV(fg, args.printcsv)

    if args.printfreqcsv:
        pprint(args)
        if not oa: oa = OA()
        if not fg: fg = FG()
        oa.printGroupsAndOutcomesToCSV(fg, args.printcsv, freqs = True)

    if args.printnumgroups:
        pprint(args)
        if not oa: oa = OA()
        if not fg: fg = FG()
        pprint(('number of groups per outcome:', og.numGroupsPerOutcome(fg, args.printcsv)))

    if args.loessplot:
        if not oa: oa = OA()
        if not fg: fg = FG()
        # whitelist = whitelist
        oa.loessPlotFeaturesByOutcome(fg, args.spearman, args.p_correction_method, blacklist, whitelist.union(args.loessplot), args.showfeatfreqs, outputdir=args.outputdir, outputname=args.outputname, topicLexicon=args.topiclexicon)

    if args.densifytable:
        if not oa: oa = OA()
        row_column, col_column, value_column = args.densifytable
        outputFile = makeOutputFilename(args, None, None, suffix="_dense.csv") if args.outputname else None
        oa.tableToDenseCsv(row_column, col_column, value_column, output_csv_filename=outputFile, compress_csv=True)

    #Correlation Analysis Options:
    correls = None
    if args.compTagcloud and args.compTCsample1 and args.compTCsample2:
        if not fg: fg = FG()
        if not oa: oa = OA()
        correls = oa.IDPcomparison(fg, args.compTCsample1, args.compTCsample2, blacklist=blacklist, whitelist=whitelist)

    if not args.compTagcloud and not args.cca and (args.correlate or args.rmatrix or args.tagcloud or args.topictc or args.corptopictc or args.barplot or args.featcorrelfilter or args.makewordclouds or args.maketopicwordclouds or args.outcomeWithOutcomeOnly or args.interactionDdla):
        if not oa: oa = OA()
        if not fg: fg = FG()
        if args.interactionDdla:
            if not dlaw: dlaw = DLAW()
            # if len(args.outcomefieldsprint "There were no features with significant interactions") > 1: raise NotImplementedError("Multiple outcomes with DDLA not yet implemented")
            # Step 1 Interaction
            if not args.outcomeinteraction:
              args.outcomeinteraction = [args.interactionDdla]
            oa = OA()
            print("##### STEP 1: Finding features with significant interaction term")
            correls = oa.correlateWithFeatures(fg, args.spearman,
                                               args.p_correction_method, args.outcomeinteraction, blacklist,
                                               whitelist, args.showfeatfreqs, args.outcomeWithOutcome, args.outcomeWithOutcomeOnly,
                                               logisticReg=args.logisticReg, outputInteraction=True, groupsWhere=args.groupswhere)
            inter_keys = [i for i in list(correls.keys()) if " * " in i]
            # correls = {outcome1: {feat: (R,p,N,freq)}}

            # whitelist should be different for multiple outcomes
            ddla_whitelists = {inter_key: [k for k, i in correls[inter_key].items() if i[1] < args.ddlaSignificance] for inter_key in inter_keys}

            correls = {"INTER["+k+"]": v for k, v in correls.items()}

            for out_name, ddla_whitelist in ddla_whitelists.items():
                if not ddla_whitelist:
                    continue
                out = out_name.split(" from ")[-1]

                whitelist = dlaw.makeBlackWhiteList(ddla_whitelist, '', [])

                # print str([i for j in correls.values() for i in j.iteritems() if i[1][0]*i[1][0] > 1])[:300]
                # exit()
                # Step 2: do correlations on both ends of the interaction variable

                print("##### STEP 2: getting correlations within groups")
                args.outcomeinteraction = []
                args.outcomefields = [out]
                og = OG()
                if args.groupswhere:
                    where = args.interactionDdla + "=1 and " + args.groupswhere
                else:
                    where = args.interactionDdla+"=1"
                print("##### WHERE: %s" % where)
                correls_1 = oa.correlateWithFeatures(fg, args.spearman,
                                                     args.p_correction_method, args.outcomeinteraction, blacklist,
                                                     whitelist, args.showfeatfreqs, args.outcomeWithOutcome, args.outcomeWithOutcomeOnly,
                                                     logisticReg=args.logisticReg, groupsWhere = where)

                correls.update({"["+k+"]_1": v for k, v in correls_1.items()})
                og = OG()
                if args.groupswhere:
                    where = args.interactionDdla + "=0 and " + args.groupswhere
                else:
                    where = args.interactionDdla+"=0"
                print("##### WHERE: %s" % where)
                correls_0 = oa.correlateWithFeatures(fg, args.spearman,
                                                     args.p_correction_method, args.outcomeinteraction, blacklist,
                                                     whitelist, args.showfeatfreqs, args.outcomeWithOutcome, args.outcomeWithOutcomeOnly,
                                                     logisticReg=args.logisticReg, groupsWhere = where)
                correls.update({"["+k+"]_0": v for k, v in correls_0.items()})

        elif args.IDP:
            correls = oa.IDP_correlate(fg, outcomeWithOutcome=args.outcomeWithOutcome, includeFreqs=args.showfeatfreqs, blacklist=blacklist, whitelist=whitelist )
        elif args.zScoreGroup:
            correls = oa.zScoreGroup(fg, outcomeWithOutcome=args.outcomeWithOutcome, includeFreqs=args.showfeatfreqs, blacklist=blacklist, whitelist=whitelist)
        elif args.auc:
            correls = oa.aucWithFeatures(fg, outcomeWithOutcome=args.outcomeWithOutcome, includeFreqs=args.showfeatfreqs, blacklist=blacklist, whitelist=whitelist, bootstrapP = args.bootstrapp, groupsWhere=args.groupswhere)
        else:
            correls = oa.correlateWithFeatures(fg, args.spearman, args.p_correction_method, args.outcomeinteraction, blacklist, whitelist, args.showfeatfreqs, args.outcomeWithOutcome, args.outcomeWithOutcomeOnly, logisticReg=(args.logisticReg or args.cohensd), cohensD=args.cohensd, outputInteraction=args.outputInteractionTerms, groupsWhere=args.groupswhere)
        if args.topicdupefilter:#remove duplicate topics (keeps those correlated more strongly)
            correls = oa.topicDupeFilterCorrels(correls, args.topiclexicon)

    if args.ccaPermute:
        if not og: og = OG()
        if not fg: fg = FG()
        cca = CCA(fg, og, args.cca)
        if args.ccaOutcomesVsControls:
            cca.ccaPermuteOutcomesVsControls(nPerms = args.ccaPermute)
        else:
            cca.ccaPermute(nPerms = args.ccaPermute)

    if args.cca:
        if not oa: oa = OA()
        if not fg: fg = FG()
        cca = CCA(fg, oa, args.cca)
        if args.ccaOutcomesVsControls:
            (featComp, outcomeComp, dVectorDict) = cca.ccaOutcomesVsControls(penaltyX = args.penaltyFeats,
                                                                penaltyZ = args.penaltyOutcomes)
        else:
            (featComp, outcomeComp, dVectorDict) = cca.cca(penaltyX = args.penaltyFeats,
                                                           penaltyZ = args.penaltyOutcomes)

        paramString = str(args)+"\n\n"
        paramString += "Components D vector\n"+'\n'.join(["%s, %.3f" % (comp,d) for comp, d in sorted(dVectorDict.items())])
        correls = featComp
        # Print csvs, topic_tagcloud
        if args.rmatrix:
            if args.outputname:
                outputFile = args.outputname
            else:
                outputFile = os.path.join(args.outputdir, makeOutputFilename(args, fg, oa, suffix="rMatrix"))

            oa.correlMatrix(featComp, outputFile+".feat", outputFormat='html', sort=args.sort, paramString=paramString.replace("\n","<br>"), nValue=args.nvalue, cInt=args.confint, freq=args.freq)
            oa.correlMatrix(outcomeComp, outputFile+".outcome", outputFormat='html', sort=args.sort, paramString=paramString.replace("\n","<br>"), nValue=args.nvalue, cInt=args.confint, freq=args.freq)
            if args.csv:
                oa.correlMatrix(featComp, outputFile+".feat", outputFormat='csv', sort=args.sort, paramString=paramString, nValue=args.nvalue, cInt=args.confint, freq=args.freq)
                oa.correlMatrix(outcomeComp, outputFile+".outcome", outputFormat='csv', sort=args.sort, paramString=paramString, nValue=args.nvalue, cInt=args.confint, freq=args.freq)
            if args.pickle:
                oa.correlMatrix(featComp, outputFile+".feat", outputFormat='pickle', sort=args.sort, paramString=paramString, nValue=args.nvalue, cInt=args.confint, freq=args.freq)
                oa.correlMatrix(outcomeComp, outputFile+".outcome", outputFormat='pickle', sort=args.sort, paramString=paramString, nValue=args.nvalue, cInt=args.confint, freq=args.freq)
            outputFile += ".feat"
        if args.savemodels:
            cca.saveModel(args.picklefile)

    if args.predictCcaCompsFromModel:
        if not og: og = OG()
        if not fg: fg = FG()
        cca = CCA(fg, og)
        if args.loadmodels:
            cca.loadModel(args.picklefile)
        features = fg.featureTable.split('$')[1]
        msgs = args.corptable.split('/')[-1].split('.')[0]
        components = cca.predictCompsToSQL(tablename=args.newSQLtable,
                                           csv = args.csv,
                                           outputname = args.outputname if args.outputname else os.path.join(args.outputdir, makeOutputFilename(args, fg, oa, suffix="rMatrix")),
                                           useXFeats = args.usexfeats, useXControls = args.usexcontrols)
    #"""
    if args.correlate:
        pprint(args)
        for outcomeField, featRs in correls.items():
            print("\n%s:" % outcomeField)
            cnt = 0
            for featR in featRs.items():
                if featR[1][1] < args.maxP: cnt +=1
            #pprint(featRs)#debug
            pprint(sorted(list(featRs.items()), key= lambda f: f[1][0] if not isnan(f[1][0]) else 0),depth=3, compact=True)
            print("\n%d features significant at p < %s" % (cnt, args.maxP))
    #"""
    if args.rmatrix and not args.cca:
        if args.outputname:
            outputFile = args.outputname
        else:
            outputFile = os.path.join(args.outputdir, makeOutputFilename(args, fg, oa, suffix="rMatrix"))

        metric = dlac.getMetric(args.logisticReg, args.cohensd, args.IDP, args.spearman, args.outcomecontrols)
        if args.auc: metric = 'AUC'
        oa.correlMatrix(correls, outputFile, outputFormat='html', sort=args.sort, paramString=str(args), nValue=args.nvalue, cInt=args.confint, freq=args.freq, metric=metric)

    if args.csv and not args.cca and correls:
        if args.outputname:
            outputFile = args.outputname
        else:
            outputFile = os.path.join(args.outputdir, makeOutputFilename(args, fg, oa, suffix="rMatrix"))

        metric = dlac.getMetric(args.logisticReg, args.cohensd, args.IDP, args.spearman, args.outcomecontrols)
        if args.auc: metric = 'AUC'
        oa.correlMatrix(correls, outputFile, outputFormat='csv', sort=args.sort, paramString=str(args), nValue=args.nvalue, cInt=args.confint, freq=args.freq, metric=metric)

    if args.tagcloud:
        metric = dlac.getMetric(args.logisticReg, args.cohensd, args.IDP, args.spearman, args.outcomecontrols)
        outputFile = makeOutputFilename(args, fg, oa, suffix="_tagcloud")
        oa.printTagCloudData(correls, args.maxP, outputFile, str(args), maxWords = args.maxtcwords, duplicateFilter = args.tcfilter, colorScheme=args.tagcloudcolorscheme, cleanCloud = args.cleancloud, metric=metric)
    if args.makewordclouds:
        if not args.tagcloud:
            print("ERROR, can't use --make_wordclouds without --tagcloud", file=sys.stderr)
            sys.exit()
        wordcloud.tagcloudToWordcloud(outputFile, withTitle=True, fontFamily="Meloche Rg", fontStyle="bold", toFolders=True, metric=metric)

    if args.topictc or args.corptopictc:
        if args.corptopictc: oa.lexicondb = oa.corpdb
        metric = dlac.getMetric(args.logisticReg, args.cohensd, args.IDP, args.spearman, args.outcomecontrols)
        outputFile = makeOutputFilename(args, fg, oa, suffix='_topic_tagcloud')
        # use plottingWhitelistPickle to link to a pickle file containing the words driving the categories
        oa.printTopicTagCloudData(correls, args.topiclexicon, args.maxP, str(args), duplicateFilter = args.tcfilter, colorScheme=args.tagcloudcolorscheme, outputFile = outputFile, useFeatTableFeats=args.useFeatTableFeats, maxWords=args.numtopicwords, cleanCloud=args.cleancloud, metric=metric, minAbsEffectSize=args.minabseffectsize)
        # don't want to base on this: maxWords = args.maxtcwords)
    if args.maketopicwordclouds:
        if not args.topictc and not args.corptopictc:
            print("ERROR, can't use --make_topic_wordclouds without --topic_tagcloud or --corp_topic_tagcloud", file=sys.stderr)
            sys.exit()
        print(outputFile)
        wordcloud.tagcloudToWordcloud(outputFile, withTitle=True, fontFamily="Meloche Rg", fontStyle="bold", toFolders=True, metric=metric, keepDuplicates=args.keepduplicates)

    comboCorrels = None
    if args.combormatrix:
        if not oa: oa = OA()
        if not fg: fg = FG()
        comboCorrels = oa.correlateControlCombosWithFeatures(fg, args.spearman, args.p_correction_method, blacklist, whitelist, args.showfeatfreqs, args.outcomeWithOutcome)
        if args.csv:
            outputStream = sys.stdout
            if args.outputname:
                outputStream = open(args.outputname+'.combocntrl.csv', 'w')
            oa.outputComboCorrelMatrixCSV(comboCorrels, outputStream, paramString = str(args))
        else:
            pprint(comboCorrels)

    if args.featcorrelfilter:
         if not fr: fr=FR()
         args.feattable = fr.createCorrelRefinedFeatTable(correls)

    if args.ddlaFiles:
        ddla = None
        if all(['.csv' in fileName for fileName in args.ddlaFiles]):
            outputname = args.outputname if args.outputname else None
            ddla = DDLA(args.ddlaFiles[0], args.ddlaFiles[1], outputname)
            ddla.differential()
        if args.ddlaTagcloud:
            correls = ddla.outputForTagclouds()
            OA().printTagCloudData(correls, args.maxP, outputFile = ddla.outputFile[:-4]+"_tagcloud", colorScheme = args.tagcloudcolorscheme)
            ## TODO : use this output into OutcomeGetter.


    ##multiRegression Options:
    coeffs = None
    if args.multir: #add any options that output multiR results to be caught here
        if not oa: oa = OA()
        if not fg: fg = FG()
        pprint(args)
        pprint(coeffs)
        coeffs = oa.multRegressionWithFeatures(fg, args.spearman, args.p_correction_method, blacklist, whitelist, args.showfeatfreqs, args.outcomeWithOutcome, interactions = args.interactions)

        if args.csv:
            outputFile = makeOutputFilename(args, fg, oa, "multiprint") ##uses outputdir
            oa.writeSignificantCoeffs4dVis(coeffs, outputFile, args.outcomefields, paramString=str(args))
        #TODO: do some sort of sorted printing?

    # Mediation Analysis
    if args.mediation:

        # more than one feature location flag
        if sum([args.feat_as_path_start, args.feat_as_outcome, args.feat_as_control, args.no_features]) > 1:
            print("You must specify only one of the following: --feat_as_path_start, --feat_as_outcome, --feat_as_control, --no_features")
            sys.exit()

        # default mode, catch no feature table or no outcome table
        if not (args.feat_as_path_start or args.feat_as_outcome or args.feat_as_control or args.no_features) and (not args.feattable or args.outcometable == dlac.DEF_OUTCOME_TABLE):
            print("You must specify a feature table (-f FEAT_TABLE) and an outcome table (--outcome_table OUTCOME_TABLE)")
            sys.exit()

        if not args.no_features:
            if args.feat_as_path_start:
                if not args.feattable:# or len(args.outcomepathstarts) == 0:
                    print("You must specify a feature table: -f FEAT_TABLE")
                    sys.exit()
                if len(args.outcomefields) == 0 or len(args.outcomemediators) == 0:
                    print("You must specify at least one mediator and outcome")
                    sys.exit()
            elif args.feat_as_outcome:
                if not args.feattable:
                    print("You must specify a feature table: -f FEAT_TABLE")
                    sys.exit()
                if len(args.outcomepathstarts) == 0 or len(args.outcomemediators) == 0:
                    print("You must specify at least one mediator and path start")
                    sys.exit()
            elif args.feat_as_control:
                if not args.feattable:
                    print("You must specify a feature table: -f FEAT_TABLE")
                    sys.exit()
                if len(args.outcomepathstarts) == 0 or len(args.outcomemediators) == 0 or len(args.outcomefields) == 0:
                    print("You must specify at least one mediator, path start and outcome")
                    sys.exit()
        else:
            if args.feattable:
                print("WARNING: You specified an feature table AND the flag --no_features. This table is being ignored.")
            if args.outcometable == dlac.DEF_OUTCOME_TABLE:
                print("You must specify an outcome table")
                sys.exit()
            if len(args.outcomepathstarts) == 0 or len(args.outcomemediators) == 0 or len(args.outcomefields) == 0:
                print("You must specify at least one mediator, path start and outcome")
                sys.exit()

        path_starts = args.outcomepathstarts
        mediators = args.outcomemediators
        outcomes = args.outcomefields

        if args.mediationboot:
            mediation_method = "boot"
        else:
            mediation_method = "parametric"

        med_switch = "default"

        if args.feat_as_path_start:
            med_switch = "feat_as_path_start"
            args.outcomefields = args.outcomemediators + args.outcomefields
            controls = args.outcomecontrols
        elif args.feat_as_outcome:
            med_switch = "feat_as_outcome"
            args.outcomefields = args.outcomepathstarts + args.outcomemediators
            controls = args.outcomecontrols
        elif args.feat_as_control:
            med_switch = "feat_as_control"
            controls = args.outcomecontrols
            args.outcomecontrols = []
            args.outcomefields = args.outcomepathstarts + args.outcomemediators + args.outcomefields
        elif args.no_features:
            med_switch = "no_features"
            args.outcomefields = args.outcomepathstarts + args.outcomemediators + args.outcomefields
            controls = args.outcomecontrols
        else:
            args.outcomefields = args.outcomepathstarts + args.outcomefields
            controls = args.outcomecontrols

        og = OG()
        fg = FG() if not args.no_features else None

        # run mediation
        mg = MediationAnalysis(fg, og, path_starts, mediators, outcomes, controls,
                method=mediation_method, boot_number=args.mediationbootnum, sig_level=args.maxP, style=args.mediation_style)
        mg.mediate(med_switch, args.p_correction_method, logisticReg=args.logisticReg)

        # print mediation results
        if args.mediationsummary: mg.print_summary(args.outputname)
        if args.mediationcsv or args.csv: mg.print_csv(args.outputname)

    ##Prediction methods:

    def printMessagesAndPredictions(dlaw, prediction_table, outcome_table=None, outcome=None, num_groups=2, num_messages=3):

        # get unique groups
        fields = ["DISTINCT {}".format(dlaw.correl_field)]
        selectQuery = dlaw.qb.create_select_query(dlaw.corptable).set_fields(fields).set_limit(num_groups)
        groups = [row[0] for row in selectQuery.execute_query()]

        # get messages, outcomes, and predictions for the groups
        dlac.warn("\nExample predictions\n-------")
        for group in groups:

            dlac.warn("Group ID: {}".format(group))
            selectQuery = dlaw.qb.create_select_query(dlaw.corptable).set_fields(["COUNT(*)"]).where("{} = {}".format(dlaw.correl_field, group))
            num_messages = min(selectQuery.execute_query()[0][0], num_messages)

            # print messages for the group
            selectQuery = dlaw.qb.create_select_query(dlaw.corptable).set_fields([dlaw.message_field]).where("{} = {}".format(dlaw.correl_field, group)).set_limit(num_messages)
            messages = [row[0] for row in selectQuery.execute_query()]
            dlac.warn("Top {} messages for the group:".format(num_messages))
            for message in messages:
                dlac.warn(message)
                dlac.warn('\n')

            selectQuery = dlaw.qb.create_select_query(prediction_table).set_fields(['*']).where("{} = {}".format(dlaw.correl_field, group))
            prediction = selectQuery.execute_query()[0][1]

            if outcome:
                selectQuery = dlaw.qb.create_select_query(outcome_table).set_fields(outcome).where("{} = {}".format(dlaw.correl_field, group))
                outcome_value = selectQuery.execute_query()[0]
                dlac.warn("Ground-truth, Prediction: {}, {}\n-------".format(outcome_value, prediction))

            else:
                dlac.warn("Prediction: {}\n-------".format(prediction))

    rp = None #regression predictor
    crp = None #combined regression predictor
    fgs = None #feature getters
    dr = None #Dimension Reducer

    if args.trainregression or args.testregression or args.combotestregression or args.predictregression or args.predictrtofeats or args.predictalltofeats or args.regrToLex or args.predictRtoOutcomeTable:
        if args.predictrtofeats and isinstance(args.low_variance_thresh, (int, float)): args.low_variance_thresh = False
        if not og: og = OG()
        if not fgs: fgs = FGs()
        if args.featureselectionstring:
            RegressionPredictor.featureSelectionString = args.featureselectionstring
        elif args.featureselection:
            RegressionPredictor.featureSelectionString = dlac.DEF_RP_FEATURE_SELECTION_MAPPING[args.featureselection]
        if args.turn_off_backoff_model:
            RegressionPredictor.backOffModel = args.model
        rp = RegressionPredictor(og, fgs, args.model, args.outlier_to_mean, n_components = args.n_components)

    if args.testcombregression:
        if not og: og = OG()
        if not fgs: fgs = FGs() #all feature getters
        crp = CombinedRegressionPredictor(og, fgs, args.combmodels, args.model)
    if args.fitreducer or args.reducertolexicon or args.supertopics or args.transformdrtofeats:
        if not og: og = OG()
        if not fg: fg = FG()
        dr = DimensionReducer(fg, args.model, og, args.n_components)
    if args.supertopics and not args.lextable:
        print("WARNING: you must use -l with --super_topics")
        sys.exit(1)
    if args.supertopics and not args.reducedlexicon and not args.reducertolexicon:
        print("WARNING: you must use --reduced_lexicon with --super_topics if not running --reducer_to_lexicon")
        sys.exit(1)

    if args.loadmodels and rp:
        rp.load(args.picklefile)

    if (args.regrToLex or args.classToLex) and '16to' in args.feattable and '16to16' not in args.feattable:
        print("WARNING: using an non 16to16 feature table")

    if args.trainregression:
        rp.train(sparse = args.sparse,  standardize = args.standardize, groupsWhere = args.groupswhere, weightedSample=args.weightedsample, outputName = args.outputname, saveFeatures = True if args.outputname else False, trainBootstraps = args.train_bootstraps, trainBootstrapsNs = args.train_bootstraps_ns)

    if args.testregression:
        rp.test(sparse = args.sparse, blacklist = blacklist,  standardize = args.standardize, groupsWhere = args.groupswhere)

    comboScores = None
    if args.combotestregression or args.controladjustreg:
        if not og: og = OG()
        if not fg: fg = FG()
        if not rp: rp = RegressionPredictor(og, fgs, args.model, args.outlier_to_mean, n_components = args.n_components)

        comboScores = None
        if args.combotestregression:
            comboScores = rp.testControlCombos(sparse = args.sparse, blacklist = blacklist,
                                           noLang=args.nolang, allControlsOnly = args.allcontrolsonly, comboSizes = args.controlcombosizes,
                                           nFolds = args.folds, savePredictions = (args.pred_csv | args.prob_csv), weightedEvalOutcome = args.weightedeval,
                                           standardize = args.standardize, residualizedControls = args.res_controls, groupsWhere = args.groupswhere,
                                           weightedSample = args.weightedsample, adaptationFactorsName = args.adaptationfactors, featureSelectionParameters=args.featureselectionparams , factorSelectionType=args.factorselectiontype, numOfFactors=args.numoffactors, pairedFactors=args.pairedfactors, outputName = args.outputname, report=args.report, integrationMethod = args.integrationmethod,
                                           stratifyFolds = args.stratifyfolds)
        elif args.controladjustreg:
            comboScores = rp.adjustOutcomesFromControls(standardize = args.standardize, sparse = args.sparse,
                                                        allControlsOnly = args.allcontrolsonly, comboSizes = args.controlcombosizes,
                                                        nFolds = args.folds, savePredictions = (args.pred_csv | args.prob_csv), groupsWhere = args.groupswhere)
        if args.pred_csv:
            outputStream = sys.stdout
            if args.outputname:
                outputStream = open(args.outputname+'.predicted_data.csv', 'w')
            RegressionPredictor.printComboControlPredictionsToCSV(comboScores, outputStream, paramString=str(args), delimiter=',')
            print("Wrote to: %s" % str(outputStream))
            if args.outputname:
                outputStream.close()
        #TODO:
        # if args.pred_feat:
        #     RegressionPredictor.printComboControlPredictionsToFeats(comboScores, label=pred_feat, paramString=str(args), delimiter='|')
        if args.csv:
            outputStream = sys.stdout
            if args.outputname:
                outputStream = open(args.outputname+'.accuracy_data.csv', 'w')
            RegressionPredictor.printComboControlScoresToCSV(comboScores, outputStream, paramString=str(args), delimiter=',')
            print("Wrote to: %s" % str(outputStream))
            if args.outputname:
                outputStream.close()
        elif not args.pred_csv:
            pprint(comboScores)


    if args.testcombregression:
        crp.test(sparse = args.sparse, standardize = args.standardize, groupsWhere = args.groupswhere)

    if args.predictregression:
        rp.predict(sparse = args.sparse, standardize = args.standardize, groupsWhere = args.groupswhere, outputName = args.outputname)

    if args.predictrtofeats and rp:
        if not fe: fe = FE()
        rp.predictToFeatureTable(sparse = args.sparse, fe = fe, name = args.predictrtofeats, standardize = args.standardize, groupsWhere = args.groupswhere)

    if args.predictRtoOutcomeTable:
        #if not fgs: fgs = FGs()
        #if not fe: fe = FE()
        rp.predictToOutcomeTable(sparse = args.sparse, name = args.predictRtoOutcomeTable, groupsWhere = args.groupswhere)

    if args.predictalltofeats and rp:
        if not fe: fe = FE()
        rp.predictAllToFeatureTable(sparse = args.sparse, fe = fe, name = args.predictalltofeats, \
                                        standardize = args.standardize, nFolds = args.folds, groupsWhere = args.groupswhere)

    if args.savemodels and rp:
        rp.save(args.picklefile)

    ##CLASSIFICATION:
    cp = None
    if args.trainclassifiers or args.testclassifiers or args.combotestclassifiers or args.predictclassifiers or args.predictctofeats or args.predictprobstofeats or args.classToLex or args.roc or args.predictCtoOutcomeTable:
        if (args.predictctofeats or args.predictprobstofeats) and isinstance(args.low_variance_thresh, (int, float)): args.low_variance_thresh = False
        if args.model == dlac.DEF_MODEL:#if model wasnt changed form a regression model
            args.model = dlac.DEF_CLASS_MODEL
        if not og: og = OG()
        if not fgs: fgs = FGs()
        if args.featureselectionstring:
            ClassifyPredictor.featureSelectionString = args.featureselectionstring
        elif args.featureselection:
            ClassifyPredictor.featureSelectionString = dlac.DEF_CP_FEATURE_SELECTION_MAPPING[args.featureselection]
        if args.turn_off_backoff_model:
            ClassifyPredictor.backOffModel = args.model
        cp = ClassifyPredictor(og, fgs, args.model, args.outlier_to_mean, n_components = args.n_components) #todo change to a method variables (like og...etc..)


    if args.loadmodels and cp:
        cp.load(args.picklefile)

    if args.trainclassifiers:
        cp.train(sparse = args.sparse, standardize = args.standardize, groupsWhere = args.groupswhere, trainBootstraps = args.train_bootstraps, trainBootstrapsNs = args.train_bootstraps_ns)

    if args.testclassifiers:
        cp.test(sparse = args.sparse, standardize = args.standardize, groupsWhere = args.groupswhere)

    comboScores = None
    if args.combotestclassifiers:
        comboScores = cp.testControlCombos(standardize = args.standardize, sparse = args.sparse, blacklist = blacklist,
                                           noLang=args.nolang, allControlsOnly = args.allcontrolsonly, comboSizes = args.controlcombosizes,
                                           nFolds = args.folds, savePredictions = (args.pred_csv | args.prob_csv),
                                           weightedEvalOutcome = args.weightedeval, stratifyFolds=args.stratifyfolds,
                                           adaptTables = args.adapttable, adaptColumns = args.adaptcolumns,
                                           groupsWhere = args.groupswhere, testFolds = args.test_folds)
        if args.csv:
            outputStream = sys.stdout
            if args.outputname:
                outputStream = open(args.outputname+'.accuracy_data.csv', 'w')
            ClassifyPredictor.printComboControlScoresToCSV(comboScores, outputStream, paramString=str(args), delimiter=',')
            print("Wrote to: %s" % str(outputStream))
            outputStream.close()
        if not (args.pred_csv or args.prob_csv):
            pprint(comboScores, compact=True)
        for outcome, cData in sorted(comboScores.items()):
            print("\n["+outcome+"]")
            mfc = 0.0
            for controls, wLangData in sorted(cData.items()):
                print("   CONTROLS -- "+' '.join(sorted(controls))) if len(controls) > 0 else print("   NO CONTROLS")
                if 0 in wLangData:
                    print("     - LANG: acc: %.3f, f1: %.3f, auc: %.3f" %\
                        tuple([wLangData[0][k] for k in ['acc', 'f1', 'auc']]))
                    mfc = wLangData[0]['mfclass_acc']
                if 1 in wLangData:
                    if len(controls) > 0:
                        print("     + LANG: acc: %.3f, f1: %.3f, auc: %.3f, auc ensemble: %.3f (p = %.4f)" %\
                              tuple([wLangData[1][k] for k in ['acc', 'f1', 'auc', 'auc_cntl_comb2', 'auc_cntl_comb2_p']]))
                    else:
                        print("     + LANG: acc: %.3f, f1: %.3f, auc: %.3f (p_vs_controls = %.4f)" %\
                              tuple([wLangData[1][k] for k in ['acc', 'f1', 'auc', 'auc_p_v_cntrls']]))
                        mfc = wLangData[1]['mfclass_acc']
            print("   (mfc_acc: %.3f)"%mfc)

        if args.pred_csv:
            outputStream = sys.stdout
            if args.outputname:
                outputStream = open(args.outputname+'.predicted_data.csv', 'w')
            ClassifyPredictor.printComboControlPredictionsToCSV(comboScores, outputStream, paramString=str(args), delimiter=',')
            print("Wrote to: %s" % str(outputStream))
            outputStream.close()
        if args.prob_csv:
            outputStream = sys.stdout
            if args.outputname:
                outputStream = open(args.outputname+'.prediction_probabilities.csv', 'w')
            ClassifyPredictor.printComboControlPredictionProbsToCSV(comboScores, outputStream, paramString=str(args), delimiter=',')
            print("Wrote to: %s" % str(outputStream))
            outputStream.close()

    if args.predictclassifiers:
        cp.predict(sparse = args.sparse, groupsWhere = args.groupswhere, probs = True)

    if args.roc:
        cp.roc(sparse = args.sparse, output_name = args.outputname if args.outputname else "ROC",  standardize = args.standardize, groupsWhere = args.groupswhere)

    if (args.predictctofeats or args.predictprobstofeats) and cp:
        if not fe: fe = FE()
        cp.predictToFeatureTable(sparse = args.sparse, fe = fe, name = args.predictctofeats or args.predictprobstofeats, groupsWhere = args.groupswhere, probs = args.predictprobstofeats)

    if args.predictCtoOutcomeTable:
        if not fgs: fgs = FGs()
        if not fe: fe = FE()
        cp.predictToOutcomeTable(sparse = args.sparse, fe = fe, name = args.predictCtoOutcomeTable, groupsWhere = args.groupswhere)

    if args.predictalltofeats and cp:
        if not fe: fe = FE()
        cp.predictAllToFeatureTable(sparse = args.sparse, fe = fe, name = args.predictalltofeats, \
                                        standardize = args.standardize, nFolds = args.folds, groupsWhere = args.groupswhere)

    c2rp = None
    if args.trainclasstoreg or args.testclasstoreg or args.predictclasstoreg:
        if not og: og = OG()
        if not fg: fg = FG()
        c2rp = ClassifyToRegressionPredictor(og, fg, modelR = args.model) #todo change to a method variables (like og...etc..

    if args.trainclasstoreg:
        c2rp.train(sparse = args.sparse, groupsWhere = args.groupswhere)

    if args.testclasstoreg:
        c2rp.test(sparse = args.sparse, groupsWhere = args.groupswhere)

    if args.predictclasstoreg:
        c2rp.predict(sparse = args.sparse, groupsWhere = args.groupswhere)

    if args.savemodels and cp:
        cp.save(args.picklefile)

    if args.loadmodels and dr:
        print("LOADING ", args.picklefile)
        dr.load(args.picklefile)

    if args.classToLex or args.regrToLex:
        lexicon_dict = None
        if rp and not cp:
            print("----- Detected a regressor")
            lexicon_dict = rp.getWeightsForFeaturesAsADict()  #returns featTable -> category -> term -> weight
        elif cp:
            print("----- Detected a classifier")
            lexicon_dict = cp.getWeightsForFeaturesAsADict()  #returns featTable -> category -> term -> weight

        lexicon_name = args.classToLex if args.classToLex else args.regrToLex
        lex_dict_with_name = dict()
        for featTableName, v in lexicon_dict.items():
            for outcome, coefs in v.items():
                if lexicon_name not in lex_dict_with_name:
                    lex_dict_with_name[lexicon_name] = dict()
                outcomes_dict = lex_dict_with_name[lexicon_name]
                if outcome not in outcomes_dict:
                    outcomes_dict[outcome] = dict()
                outcomes_dict[outcome].update(coefs)
        for lexName, lexicon in lex_dict_with_name.items():
            lex = lexInterface.WeightedLexicon(lexicon, mysql_config_file=args.mysqlconfigfile)
            lex.createWeightedLexiconTable('dd_'+lexName)

    if args.fitreducer:
        #dr.fit(sparse = args.sparse, blacklist = blacklist)
        dr.fit(sparse = args.sparse, standardize = args.standardize)

    if args.transformdrtofeats and dr:
        if not fe: fe = FE()
        dr.transform(sparse = args.sparse, standardize = args.standardize, writeToFeats = args.transformdrtofeats, fe = fe)
        
    if args.reducertolexicon or args.supertopics:
        if args.reducertolexicon:
          lexicons = dr.modelToLexicon()
          for outcomeName, lexDict in lexicons.items():
              print("ON: ", outcomeName)
              lexiconName = args.reducertolexicon if args.reducertolexicon else args.reducedlexicon
              if outcomeName != 'noOutcome':
                  lexiconName += '_'+outcomeName
              lexicon = lexInterface.WeightedLexicon(lexDict, mysql_config_file=args.mysqlconfigfile)
              lexicon.createLexiconTable(lexiconName)
              if args.supertopics:
                  lexicon.createSuperTopicTable(args.supertopics, lexiconName, args.lextable)
        else:
            lexicon = lexInterface.WeightedLexicon("", mysql_config_file=args.mysqlconfigfile)
            lexicon.createSuperTopicTable(args.supertopics, args.reducedlexicon, args.lextable)

    if args.savemodels and dr:
        dr.save(args.picklefile)

    if args.predictRtoOutcomeTable or args.predictCtoOutcomeTable:
        if not dlaw: dlaw = DLAW()
        prediction_table = "p_{}${}".format(
            rp.modelName[:4] if args.predictRtoOutcomeTable else cp.modelName[:4],
            args.predictRtoOutcomeTable if args.predictRtoOutcomeTable else args.predictCtoOutcomeTable)
        printMessagesAndPredictions(dlaw, prediction_table, args.outcometable, args.outcomefields)

    ##Plot Actions:
    if args.makealltopicwordclouds:
        outputFile = makeOutputFilename(args, None, None, suffix="_alltopics/")
        if args.tagcloudcolorscheme == 'multi':#make sure not to use multi
            args.tagcloudcolorscheme = 'blue'
        #wordcloud.makeLexiconTopicWordclouds(lexdb=args.lexicondb, lextable=args.topiclexicon, output=outputFile, color=args.tagcloudcolorscheme, max_words=args.numtopicwords, cleanCloud=args.cleancloud,mysql_config_file=args.mysqlconfigfile)
        if not dlaw: dlaw = DLAW()
        lextable = dlaw.load_lexicon(args.topiclexicon)
        wordcloud.makeLexiconTopicWordclouds(dlaw, lextable=lextable, output=outputFile, color=args.tagcloudcolorscheme, max_words=args.numtopicwords, cleanCloud=args.cleancloud)
    if args.barplot:
        outputFile = makeOutputFilename(args, fg, oa, "barplot")
        oa.barPlot(correls, outputFile)

    if args.descplot:
        if not og: og=OG()
        (groups, outcome_to_gid_to_value, controls) = og.getGroupsAndOutcomes()
        outcome_to_values = dict(  [(k_v[0], list(k_v[1].values())) for k_v in list(outcome_to_gid_to_value.items())]  )
        outputFile = makeOutputFilename(args, None, og, "desc_stats")
        from dlatk.lib.descStats import StatsPlotter
        sp = StatsPlotter(args.corpdb)
        sp.plotDescStats(outcome_to_values, len(groups), outputFile)

    if args.scatterplot: # or args.hist2d:
        scatter_dict_1 = {}
        scatter_dict_2 = {}

        if not og: og = OG()
        (out_groups, outcome_to_gid_to_value, controls) = og.getGroupsAndOutcomes()
        scatter_dict_1 = outcome_to_gid_to_value

        if args.featnames and args.feattable:
            if not fg: fg = FG()
            where = 'group_id IN ( %s )'%(','.join(map(str, out_groups)))
            (feat_groups, feature_to_gid_to_value) = fg.getGroupsAndFeats( where )
            scatter_dict_2 = feature_to_gid_to_value
        else:
            scatter_dict_2 = outcome_to_gid_to_value

        from dlatk.lib.descStats import StatsPlotter
        sp = StatsPlotter()
        for scatter_group_1 in scatter_dict_1:
            for scatter_group_2 in scatter_dict_2:
                if scatter_group_1 != scatter_group_2 or (args.featnames and args.feattable):
                    (xValues, yValues) = alignDictsAsLists(scatter_dict_1[scatter_group_1], scatter_dict_2[scatter_group_2])
                    if args.scatterplot:
                        outputFile = '%s/plots/scatter/%s.%s.%s.%s.scat'%(args.outputdir, fg.featureTable if fg else 'noFeatureTable', scatter_group_2, og.outcome_table, scatter_group_1)
                        if args.outputname: outputFile = '.'.join([args.outputname, scatter_group_1, scatter_group_2, 'scat'])
                        sp.plotScatter(scatter_group_1, xValues, scatter_group_2, yValues, outputFile)
                    if args.hist2d:
                        outputFile = '%s/plots/scatter/%s.%s.%s.%s.hist2d'%(args.outputdir, fg.featureTable if fg else 'noFeatureTable', scatter_group_2, og.outcome_table, scatter_group_1)
                        if args.outputname: outputFile = outputFile = '.'.join([args.outputname, scatter_group_1, scatter_group_2, 'hist2d'])
                        sp.plot2dHist(scatter_group_1, xValues, scatter_group_2, yValues, outputFile)
                    outputFile = '%s/plots/scatter/%s.%s.%s.%s.desc.%s'%(args.outputdir, fg.featureTable if fg else 'noFeatureTable', scatter_group_2, og.outcome_table, scatter_group_1, scatter_group_1)
                    if args.outputname: outputFile = '.'.join([args.outputname, scatter_group_1, scatter_group_2, 'desc'])
                    sp.plotDescStats(dict({scatter_group_1:xValues}), outputFile)
                    outputFile = '%s/plots/scatter/%s.%s.%s.%s.desc.%s'%(args.outputdir, fg.featureTable if fg else 'noFeatureTable', scatter_group_2, og.outcome_table, scatter_group_1, scatter_group_2)
                    if args.outputname: outputFile = '.'.join([args.outputname, scatter_group_1, scatter_group_2, 'desc'])
                    sp.plotDescStats(dict({scatter_group_2:yValues}), outputFile)

    #Analysis using only feature tables:
    if args.ttestfeats:
        if not fgs: fgs = FGs()
        #TODO: change below to use every pair of FGs (in case there are more than 2)
        results = fgs[0].ttestWithOtherFG(fgs[1], maskTable = args.masktable, groupFreqThresh=args.groupfreqthresh)
        if args.csv:
            #add this
            #can use something similar to OG correlMatrix
            pass
        pprint(results)

    if init_args.toinitfile:
      with open(init_args.toinitfile, 'w') as init_file:
        init_file.write("[constants]\n")

        if (args.dbengine and args.dbengine != dlac.DB_TYPE): init_file.write("db_type = " + str(args.dbengine)+"\n")
        if (args.corpdb and args.corpdb != dlac.DEF_CORPDB): init_file.write("corpdb = " + str(args.corpdb)+"\n")
        if (args.corptable and args.corptable != dlac.DEF_CORPTABLE): init_file.write("corptable = " + str(args.corptable)+"\n")
        if (args.correl_field): init_file.write("correl_field = " + str(args.correl_field)+"\n")
        if (args.mysqlconfigfile and args.mysqlconfigfile != dlac.MYSQL_CONFIG_FILE): init_file.write("mysql_config_file = " + str(args.mysqlconfigfile)+"\n")
        if (args.message_field and args.message_field != dlac.DEF_MESSAGE_FIELD): init_file.write("message_field = " + str(args.message_field)+"\n")
        if (args.messageid_field and args.messageid_field != dlac.DEF_MESSAGEID_FIELD): init_file.write("messageid_field = " + str(args.messageid_field)+"\n")
        if (args.encoding and args.encoding != dlac.DEF_ENCODING): init_file.write("encoding = " + str(args.encoding)+"\n")
        if (args.lexicondb and args.lexicondb != dlac.DEF_LEXICON_DB): init_file.write("lexicondb = " + str(args.lexicondb)+"\n")
        if (args.feattable and args.feattable != dlac.DEF_FEAT_TABLE):
            if isinstance(args.feattable, str):
                init_file.write("feattable = " + args.feattable+"\n")
            else:
                init_file.write("feattable = " + ", ".join([str(ftable) for ftable in args.feattable])+"\n")
        if (args.featnames and args.featnames != dlac.DEF_FEAT_NAMES): init_file.write("featnames = " + ", ".join([str(feat) for feat in args.featnames])+"\n")
        if (args.date_field and args.date_field != dlac.DEF_DATE_FIELD): init_file.write("date_field = " + str(args.date_field)+"\n")
        if (args.outcometable and args.outcometable != dlac.DEF_OUTCOME_TABLE): init_file.write("outcometable = " + str(args.outcometable)+"\n")
        if (args.outcomefields and args.outcomefields != dlac.DEF_OUTCOME_FIELDS): init_file.write("outcomefields = " + ", ".join([str(out) for out in args.outcomefields])+"\n")
        if (args.outcomecontrols and args.outcomecontrols != dlac.DEF_OUTCOME_CONTROLS): init_file.write("outcomecontrols = " + ", ".join([str(out) for out in args.outcomecontrols])+"\n")
        if (args.outcomeinteraction and args.outcomeinteraction != dlac.DEF_OUTCOME_CONTROLS): init_file.write("outcomeinteraction = " + ", ".join([str(out) for out in args.outcomeinteraction])+"\n")
        if (args.featlabelmaptable and args.featlabelmaptable != ''): init_file.write("featlabelmaptable = " + str(args.featlabelmaptable)+"\n")
        if (args.featlabelmaplex and args.featlabelmaplex != ''): init_file.write("featlabelmaplex = " + str(args.featlabelmaplex)+"\n")
        if (args.wordTable): init_file.write("wordTable = " + str(args.wordTable)+"\n")
        if (args.outputname): init_file.write("outputname = " + str(args.outputname)+"\n")
        if (args.groupfreqthresh and args.groupfreqthresh != int(dlac.DEF_GROUP_FREQ_THRESHOLD)): init_file.write("groupfreqthresh = " + str(args.groupfreqthresh)+"\n")
        if (args.lextable): init_file.write("lextable = " + str(args.lextable)+"\n")
        if (args.p_correction_method and args.p_correction_method != dlac.DEF_P_CORR): init_file.write("p_correction_method = " + str(args.p_correction_method)+"\n")
        if (args.tagcloudcolorscheme and args.tagcloudcolorscheme != 'multi'): init_file.write("tagcloudcolorscheme = " + str(args.tagcloudcolorscheme)+"\n")
        if (args.maxP and args.maxP != float(dlac.DEF_P)): init_file.write("maxP = " + str(args.maxP)+"\n")
        if (args.model and args.model != dlac.DEF_MODEL): init_file.write("model = " + str(args.model)+"\n")

        init_file.close()
    
    warning = '''-------
Settings:

Database - {}
Corpus - {}
Group ID - {}'''.format(args.corpdb, args.corptable, args.correl_field)
    dlac.warn(warning) 
    if args.feattable:
        dlac.warn("Feature table(s) - {}".format(args.feattable))
    if args.outcometable:
        dlac.warn("Outcome table - {}".format(args.outcometable))
    if args.outcomefields:
        dlac.warn("Outcome(s) - {}".format(' '.join(args.outcomefields)))
    if args.outcomecontrols:
        dlac.warn("Control(s) - {}".format(' '.join([args.outcomecontrols[index] for index in range(min(3, len(args.outcomecontrols)))])))

    dlac.warn("-------\nInterface Runtime: %.2f seconds"% float(time.time() - start_time))
    dlac.warn("DLATK exits with success! A good day indeed  ¯\_(ツ)_/¯.")

if __name__ == "__main__":
    main()
    sys.exit(0)