-
Notifications
You must be signed in to change notification settings - Fork 3
/
article-information-2019.tex
1524 lines (1249 loc) · 147 KB
/
article-information-2019.tex
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
% CC by 4.0
% TODO:
% EDITING:
% - Structure ------------------------- x
% - External links -------------------- x
% - Internal links -------------------- x
% - Sections/subsections/appendices - x
% - References ---------------------- x
% - Tables/figures ------------------ x
% - footnotes ----------------------- x
% - Abbreviations --------------------- x
% - Bibliography --------------------- x
% - Tables/figures -------------------- x
% - Notation
% - Grammar and spelling
% - Hyphenation
% CONTENT:
% - sim data: add error metrics, confirm statements about results - x
% - sim data xnn: update figure: connections, feed-forward direction, make subfigure reference match discrimination measure table - x
% - mort data: add error metrics, confirm statements about results - x
% - mort data: update figure: connections, feed-forward direction, make subfigure reference match discrimination measure table - x
% - make subfigure reference match XNN charts - x
%=================================================================
%%%\documentclass[information,article,submit,moreauthors,pdftex]{definitions/mdpi}
\documentclass[information,article,accept,moreauthors,pdftex]{Definitions/mdpi}
\usepackage{soul}
%=================================================================
\firstpage{1}
\makeatletter
\setcounter{page}{\@firstpage}
\makeatother
\pubvolume{xx}
\issuenum{1}
\articlenumber{5}
\pubyear{2020}
\copyrightyear{2020}
%\externaleditor{Academic Editor: name}
\history{Received: 21 December 2019; Accepted: 25 February 2020; Published: date}
%\updates{yes} % If there is an update available, un-comment this line
%=================================================================
% Full title of the paper (Capitalized)
\Title{A Responsible Machine Learning Workflow\\{with Focus on Interpretable Models, Post-hoc Explanation, and Discrimination Testing}}%Attention AE/ME. The following layout issues have not been checked by the English Editing Department and must be carefully verified by the AE/Layout Department: All callout issues, bold usage of callouts, and references to callouts in the text. Correct callout usage in figures. Figure and Table layout issues. Footnote formatting and Glossaries have not been checked. En dash usage for negative values, en dash usage to indicate relationships, en dash usage to indicate bonds (especially in chemistry). The English Editing Department is not responsible for correct italic usage for genes, proteins and technical terminology. This responsibility belongs to the authors. The following are also not checked: spacing between numbers and units of measurement, ratios, en dashes for ranges, date and time formats, punctuation in equation lines, and less than/more than spacing (< >). Finally, capitalization and layout of titles/headings must be properly checked as well as ensuring 'Eq.' and 'Fig.' are properly spelled out, as these are layout issues.
% Authors, for the paper (add full first names)
\Author{{Navdeep Gill} $^{1, \dagger}$, Patrick Hall $^{1,2,}$*$^{,\dagger}$, Kim Montgomery $^{1,\dagger}$ and Nicholas Schmidt $^{3,}$*$^{,\dagger}$} %%%Author order is not the same as on the system, please confirm which one is correct.
%Please carefully check the accuracy of names and affiliations.
% Authors, for metadata in PDF
\AuthorNames{Navdeep Gill, Patrick Hall, Kim Montgomery and Nicholas Schmidt}
% Affiliations / Addresses (Add [1] after \address if there is only one affiliation.)
\address{%
$^{1}$ \quad {H2O.ai}, Mountain View 94043, CA, USA; navdeep.gill@h2o.ai (N.G.); kim.montgomery@h2o.ai (K.M.)\\
$^{2}$ \quad {Department of Decision Sciences, The George Washington University}, Washington 20052, DC, USA\\%%%Please provide the department, zip code, city, country.
$^{3}$ \quad {BLDS, LLC}, Philadelphia 19103, PA, USA}
% Contact information of the corresponding author
\corres{Correspondence: phall@h2o.ai (P.H.); nschmidt@bldsllc.com (N.S.)}
% Current address and/or shared authorship
\firstnote{All authors contributed equally to this work.}
% Abstract (Do not insert blank lines, i.e. \\)
\abstract{This manuscript outlines a viable approach for training and evaluating machine learning systems for high-stakes, human-centered, or regulated applications using common Python programming tools. The accuracy and intrinsic interpretability of two types of constrained models, monotonic gradient boosting machines and explainable neural networks, a deep learning architecture well-suited for structured data, are assessed on simulated data and publicly available mortgage data. For maximum transparency and the potential generation of personalized adverse action notices, the constrained models are analyzed using post-hoc explanation techniques including plots of partial dependence and individual conditional expectation and with global and local Shapley feature importance. The constrained model predictions are also tested for disparate impact and other types of discrimination using measures with long-standing legal precedents, adverse impact ratio, marginal effect, and standardized mean difference, along with straightforward group fairness measures. By combining interpretable models, post-hoc explanations, and discrimination testing with accessible software tools, this text aims to provide a template workflow for machine learning applications that require high accuracy and interpretability and that mitigate risks of discrimination.}
% Keywords
\keyword{deep learning; disparate impact; explanation; fairness; gradient boosting machine; interpretable; machine learning; neural network; Python}
% for subtables
\usepackage{caption}
\usepackage{graphicx}
\usepackage{subcaption}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\setcounter{section}{0}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction}
Responsible artificial intelligence (AI) has been variously conceptualized as AI-based products or projects that use transparent technical mechanisms, that create appealable decisions or outcomes, that~perform reliably and in a trustworthy manner over time, that exhibit minimal social discrimination, and that are designed by humans with diverse experiences, both in terms of demographics and professional backgrounds (e.g.,
% THESE HYPERLINKS ARE NECESSARY - PH
\href{https://ec.europa.eu/jrc/communities/sites/jrccties/files/03_dignum_v.pdf}
{Responsible Artificial Intelligence},
\href{https://www.accenture.com/_acnmedia/pdf-92/accenture-afs-responsible-ai.pdf}
{\textit{Responsible AI: A Framework for Building Trust in Your AI Solutions}},
\href{https://www.pwc.com/gx/en/issues/data-and-analytics/artificial-intelligence/what-is-responsible-ai.html}
{PwC's Responsible AI},
\href{https://ai.google/responsibilities/responsible-ai-practices/}
{Responsible AI Practices}).
Although responsible AI is today a somewhat broad and amorphous notion, at least one aspect is becoming clear. Machine learning (ML) models, a common application of AI, can present serious risks. ML~models can be inaccurate and unappealable %INCORRECT - PH -- Please confirm that the intended meaning has been retained.
black-boxes, even with the application of newer post-hoc explanation techniques \cite{please_stop} (e.g.,
\href{https://www.nytimes.com/2017/06/13/opinion/how-computers-are-harming-criminal-justice.html}
{\textit{When a Computer Program Keeps You in Jail}}). ML models can perpetuate and exacerbate discrimination \cite{feldman2015certifying,dwork2012fairness,gender_shades}, and ML models can be hacked, resulting in manipulated model outcomes or the exposure of proprietary intellectual property or sensitive training data \cite{security_of_ml,model_stealing,membership_inference,shokri2019privacy}. This~manuscript makes no claim that these interdependent issues of ML opaqueness, discrimination, privacy harms, and security vulnerabilities have been resolved, even as singular entities, and much less as complex intersectional phenomena. However, Sections \ref{sec:m_and_m}--\ref{sec:disc} do propose some specific technical countermeasures, mostly in the form of interpretable models, post-hoc explanation, and disparate impact (DI) and discrimination testing, that responsible practitioners can use to address a subset of these vexing problems. (In the United States (US), interpretable models, explanations, DI testing, and the model documentation they enable may also be required under the Civil Rights Acts of 1964 and 1991, the Americans with Disabilities Act, the Genetic Information Nondiscrimination Act, the Health Insurance Portability and Accountability Act, the Equal Credit Opportunity Act (ECOA), the Fair Credit Reporting Act (FCRA), the Fair Housing Act, Federal Reserve Supervision and Regulation (SR) Letter 11-7, the European Union (EU) General Data Protection Regulation (GDPR) Article 22, or other laws and regulations \cite{ff_interpretability}. Note that this text and associated software are not, and should not be construed as, legal advice or regulatory compliance requirements.)
Section \ref{sec:m_and_m} describes methods and materials, including training datasets, interpretable and constrained models, post-hoc explanations, tests for DI and other social discrimination, and public and open source software resources associated with this text. In Section \ref{sec:res}, interpretable and constrained modeling results are compared to less interpretable and unconstrained models, and~post-hoc explanation and discrimination testing results are also presented for interpretable models. Of course, an even wider array of tools and techniques are likely helpful to fully minimize discrimination, inaccuracy, privacy, and security risks associated with ML models. Section \ref{sec:disc} puts forward a more holistic responsible ML modeling workflow, and addresses the burgeoning Python ecosystem for responsible AI, along with appeal and override of automated decisions, and discrimination testing and remediation in practice. Section \ref{sec:con} closes this manuscript with a brief summary of the outlined methods, materials, results, and discussion.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Materials and Methods}\label{sec:m_and_m}
The simulated data (see Section~\ref{ssec:sim_data}) are based on the well-known Friedman datasets. Its~known feature importance and augmented discrimination characteristics are used to gauge the validity of interpretable modeling, post-hoc explanation, and discrimination testing techniques \cite{friedman1979tree,friedman1991multivariate}. The~mortgage data (see Section~\ref{ssec:mort_data}) are sourced from the Home Mortgage Disclosure Act (HMDA) database, a fairly realistic data source for demonstrating the template workflow \cite{hmda} (see
\href{https://www.consumerfinance.gov/data-research/hmda/}
{Mortgage data (HMDA)}). %%% footnote is not allowed in our jounral, we have moved them in the main text, please confirm if you are OK with this.
To provide a sense of fit differences, performance is compared on simulated data and collected mortgage data between the more interpretable constrained ML models and the less interpretable unconstrained ML models. Because the unconstrained ML models, gradient boosting machines (GBMs, e.g., \cite{friedman2001greedy,esl}) and artificial neural networks (ANNs, e.g., \cite{recht2011hogwild,hinton2012improving,sutskever2013importance,zeiler2012adadelta}), do not exhibit convincing accuracy benefits on the simulated or mortgage data and can also present the unmitigated risks discussed above, further explanation and discrimination analyses are applied only to the constrained, interpretable ML models \cite{please_stop,fair_washing,scaffolding}. Here, monotonic gradient boosting machines
(MGBMs, as implemented in
%\href{https://xgboost.readthedocs.io/en/latest/tutorials/monotonic.html}
\texttt{XGBoost} or
%\href{https://github.com/h2oai/h2o-3/blob/master/h2o-py/demos/H2O_tutorial_gbm_monotonicity.ipynb}
\texttt{h2o}, see Section~\ref{ssec:mgbm})
and explainable neural networks (XNNs, e.g., \cite{,wf_xnn,yang2019enhancing}, see Section~\ref{ssec:xnn}) will serve as those more interpretable models for subsequent explanatory and discrimination analyses. MGBM and XNN interpretable model architectures are selected for the example workflow because they are straightforward variants of popular unconstrained ML models. If~practitioners are working with GBM and ANN models, it should be relatively uncomplicated to also evaluate the constrained versions of these models.
The same can be said of the selected explanation methods and discrimination tests. Due to their post-hoc nature, they can often be shoe-horned into existing ML workflows and pipelines. Presented explanation techniques include partial dependence (PD) and individual conditional expectation (ICE) (see Section~\ref{ssec:pd_ice}) and Shapley values (see Section~\ref{ssec:shap}) \cite{esl,ice_plots,shapley,tree_shap}. PD, ICE, and Shapley values provide direct, global, and local summaries and descriptions of constrained models without resorting to the use of intermediary and approximate surrogate models. Discrimination testing methods discussed (see Section~\ref{ssec:di}) include adverse impact ratio
(AIR, see Part 1607---Uniform Guidelines on Employee Selection Procedures (1978) \textsection 1607.4),
marginal effect (ME), and standardized mean difference (SMD) \cite{feldman2015certifying,cohen1988statistical,cohen1992power}.
%%% footnote is not allowed in our jounral, we have moved them in the main text, please confirm if you are OK with this.
Accuracy and other confusion matrix measures are also reported by demographic segment \cite{zafar2017fairness}. All outlined materials and methods are implemented in open source Python code, and are made available on GitHub (see Section~\ref{ssec:soft}).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Simulated Data}\label{ssec:sim_data}
Simulated data are created based on a signal-generating function, $f$, applied to input data, $\mathbf{X}$, first~proposed in \citet{friedman1979tree} and extended in \citet{friedman1991multivariate}:
\begin{equation}
\begin{aligned}\label{eq:sim}\small
f(\mathbf{X}) = 10\sin(\pi~X_{\text{Friedman},1}~X_{\text{Friedman},2}) + 20(X_{\text{Friedman},3} - 0.5)^2 + 10~X_{\text{Friedman},4} + 5~X_{\text{Friedman},5}
\end{aligned}
\end{equation}
where each $X_{\text{Friedman},j}$ is a random uniform feature in $[0,1]$. In Friedman's texts, a Gaussian noise term was added to create a continuous output feature for testing spline regression methodologies. In this manuscript, the signal-generating function and input features are modified in several ways. Two binary features, a categorical feature with five discrete levels, and a bias term are introduced into $f$ to add a degree of complexity that may more closely mimic real-world settings.
%When generating the data, random draws for each observation are constructed such that the features can be correlated with each other, some highly so. For example, X1 and X3 have a Pearson correlation coefficient equal to 0.58.
For binary classification analysis, the Gaussian noise term is replaced with noise drawn from a logistic distribution and coefficients are re-scaled to be $\frac{1}{5}$ of the size of those used by Friedman, and any $f(\mathbf{X})$ value above 0 is classified as a positive outcome, while $f(\mathbf{X})$ values less than or equal to zero are designated as negative outcomes. Finally, $f$ is augmented with two hypothetical protected class-control features with known dependencies on the binary outcome to allow for discrimination testing. The simulated data are generated to have eight input features, twelve after numeric encoding of categorical features, and a binary outcome, two class-control features, and 100,000 instances. The simulated data are then split into a training and test set, with 80,000 and 20,000 instances, respectively. Within the training set, a~five-fold cross-validation indicator is used for training all models. For an exact specification of the simulated data, see the software resources referenced in Section~\ref{ssec:soft}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Mortgage Data}\label{ssec:mort_data}
The mortgage dataset analyzed here is a random sample of consumer-anonymized loans from the HDMA database. These loans are a subset of all originated mortgage loans in the 2018 HMDA data that were chosen to represent a relatively comparable group of consumer mortgages. A selection of features is used to predict whether a loan is \textit{high-priced}, i.e., the annual percentage rate (APR) charged was 150 basis points (1.5\%) or more above a survey-based estimate of other similar loans offered around the time of the given loan. After data cleaning and preprocessing to encode categorical features and create missing markers, the mortgage data contain ten input features and the binary outcome, \textit{high-priced}. The data are split into a training set with 160,338 loans and a marker for 5-fold cross-validation and a test set containing 39,662 loans. While lenders would almost certainly use more information than the selected features to determine whether to offer and originate a high-priced loan, the selected input features (loan to value (LTV) ratio, debt to income (DTI) ratio, property value, loan amount, introductory interest rate, customer income, etc.) are likely to be some of the most influential factors that a lender would consider. See the resources put forward in Section \ref{ssec:soft} and Appendix \ref{a_sec:mort} for more information regarding the HMDA mortgage data.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Monotonic Gradient Boosting Machines}\label{ssec:mgbm}
MGBMs constrain typical GBM training to consider only tree splits that obey user-defined positive and negative monotonicity constraints, with respect to each input feature, $X_j$, and a target feature, $\mathbf{y}$, independently. An MGBM remains an additive combination of $B$ trees trained by gradient boosting, $T_b$, and each tree learns a set of splitting rules that respect monotonicity constraints, $\Theta^\text{mono}_b$. For an instance, $\mathbf{x}$, a trained MGBM model, $g^{\text{MGBM}}$, takes the form:
\begin{equation}
\begin{aligned}\label{eq:gbm}
g^{\text{MGBM}}(\mathbf{x}) &= \sum_{b=0}^{B-1} T_b\left(\mathbf{x}; \Theta^\text{mono}_b\right)
\end{aligned}
\end{equation}
As in unconstrained GBM, $\Theta^{\text{mono}}_b$ is selected in a greedy, additive fashion by minimizing a regularized loss function that considers known target labels, the predictions of all subsequently trained trees in $g^{\text{MGBM}}$, and the $b$-th tree splits applied to $\mathbf{x}$, $T_b(\mathbf{x};\Theta^\text{mono}_b)$, in a numeric loss function (e.g., squared loss, Huber loss), and a regularization term that penalizes complexity in the current tree. See~Appendices \ref{a_ssec:not} and \ref{a_ssec:mgbm} for details pertaining to MGBM training.
Herein, two $g^\text{MGBM}$ models are trained. One on the simulated data and one on the mortgage data. In both cases, positive and negative monotonic constraints for each $X_j$ are selected using domain knowledge, random grid search is used to determine other hyperparameters, and five-fold cross-validation and test partitions are used for model assessment. For exact parameterization of the two $g^\text{MGBM}$ models, see the software resources referenced in Section~\ref{ssec:soft}.
\subsection{Explainable Neural Networks}\label{ssec:xnn}
XNNs are an alternative formulation of additive index models in which the ridge functions are neural networks \cite{wf_xnn}. XNNs also bear a strong resemblance to generalized additive models (GAMs) and so-called explainable boosting machines (EBMs or GA\textsuperscript{2}Ms), which consider main effects and a small number of two-way interactions and may also incorporate boosting into their training \cite{esl,ga2m}. XNNs enable users to tailor interpretable neural network architectures to a given prediction problem and to visualize model behavior by plotting ridge functions. A trained XNN function, $g^{\text{XNN}}$, applied to some instance , $\mathbf{x}$, is defined as:
\begin{equation}
\begin{aligned}
\label{eq:xnn}
g^{\text{XNN}}(\mathbf{x}) = \mu_0 + \sum_{k=0}^{K-1}\gamma_k n_k(\sum^{J-1}_{j=0}\beta_{k, j}x_j)
\end{aligned}
\end{equation}
where $\mu_0$ is a global bias for $K$ individually specified ANN subnetworks, $n_k$, with weights $\gamma_k$. The~inputs to each $n_k$ are themselves a linear combination of the $J$ modeling inputs and their associated $\beta_{k,j}$ coefficients in the deepest, i.e., {projection}, layer of $g^\text{XNN}$.
Two $g^\text{XNN}$ models are trained by mini-batch stochastic gradient descent (SGD) on the simulated data and mortgage data. Each $g^\text{XNN}$ is assessed in five training folds and in a test data partition. $L_1$ regularization is applied to network weights to induce a sparse and interpretable model, where~each $n_k$ and corresponding $\gamma_k$ are ideally associated with an important $X_j$ or combination thereof. $g^\text{XNN}$ models appear highly sensitive to weight initialization and batch size. Be aware that $g^{XNN}$ architectures may require manual and judicious feature selection due to long training times. For more details regarding $g^\text{XNN}$ training, see the software resources in Section~\ref{ssec:soft} and Appendices \ref{a_ssec:not} and \ref{a_ssec:xnn}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{One-Dimensional Partial Dependence and Individual Conditional Expectation}\label{ssec:pd_ice}
PD plots are a widely-used method for describing and plotting the estimated average prediction of a complex model, $g$, across some partition of data, $\mathbf{X}$, for some interesting input feature, $X_j \in \mathbf{X}$ \cite{esl}. ICE plots are a newer method that describes the local behavior of $g$ with regard to values of an input feature in a single instance, $x_j$. PD and ICE can be overlaid in the same plot to create a holistic global and local portrait of the predictions for some $g$ and $X_j$ \cite{ice_plots}. When $\text{PD}(X_j, g)$ and $\text{ICE}(x_j, g)$ curves diverge, such plots can also be indicative of modeled interactions in $g$ or expose flaws in PD estimation, e.g., inaccuracy in the presence of strong interactions and correlations \cite{ice_plots,ale_plot}. For details regarding the calculation of one-dimensional PD and ICE, see the software resources in Section~\ref{ssec:soft} and Appendices~\ref{a_ssec:not} and \ref{a_ssec:pd_ice}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Shapley Values}\label{ssec:shap}
Shapley explanations are a class of additive, locally accurate feature contribution measures with long-standing theoretical support \cite{shapley,shapley1988shapley}. Shapley explanations are the only known locally accurate and globally consistent feature contribution values, meaning that Shapley explanation values for input features always sum to the model's prediction, $g(\mathbf{x})$, for any instance $\mathbf{x}$, and that Shapley explanation values should not decrease in magnitude for some instance of $x_j$ when $g$ is changed such that $x_j$ truly makes a stronger contribution to $g(\mathbf{x})$ \cite{shapley,tree_shap}. Shapley values can be estimated in different ways, many of which are intractable for datasets with large numbers of input features. Tree~Shapley Additive Explanations (SHAP) is a specific implementation of Shapley explanations that relies on traversing internal decision tree structures to efficiently estimate the contribution of each $x_j$ for some $g(\mathbf{x})$ \cite{tree_shap}. Tree SHAP has been implemented in popular gradient boosting libraries such as
%\href{http://docs.h2o.ai/h2o/latest-stable/h2o-py/docs/modeling.html#h2ogradientboostingestimator}
{\texttt{{h2o}}},
%\href{https://lightgbm.readthedocs.io/en/latest/Python-Intro.html}
{\texttt{{LightGBM}}}, and
%\href{https://xgboost.readthedocs.io/en/latest/python/python_intro.html}
{\texttt{{XGBoost}}}, and Tree SHAP is used to calculate accurate and consistent global and local feature importance for MGBM models in Section~\ref{ssec:int_ex_mort} and Appendix \ref{a_ssec:int_ex_sim}. Deep SHAP is an approximate Shapley value technique that creates SHAP values for ANNs \cite{shapley}. Deep SHAP is implemented in the
%\href{https://github.com/slundberg/shap}
{\texttt{{shap}}} package and is used to generate SHAP values for the two $g^{XNN}$ models discussed in Section~\ref{ssec:int_ex_mort} and Appendix \ref{a_ssec:int_ex_sim}. For more information pertaining to the calculation of Shapley values, see Appendices \ref{a_ssec:not} and \ref{a_ssec:shap}.%% we have removed the link of text, and keep web link
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Discrimination Testing Measures}\label{ssec:di}
Because many current technical discussions of fairness in ML appear inconclusive (e.g.,
\href{https://www.youtube.com/watch?v=jIXIuYdnyyk}{Tutorial: 21 Fairness Definitions and Their Politics}), this text will draw on regulatory and legal standards that have been used for years in regulated, high-stakes employment and financial decisions. The~discussed measures are also representative of fair lending analyses and pair well with the mortgage data. (See~Appendix \ref{a_sec:types} for a brief discussion regarding different types of discrimination in US legal and regulatory settings, and Appendix \ref{a_sec:prac_v_stat} for remarks on practical vs. statistical significance for discrimination measures.) One such common measure of DI used in US litigation and regulatory settings is ME. ME is simply the difference between the percent of the control group members receiving a favorable outcome and the percent of the protected class members receiving a favorable outcome:
\begin{equation}
\label{eq:me}
\begin{aligned}
\text{ME} \equiv 100 \cdot (\text{Pr}(\hat{\mathbf{y}} = 1| X_c = 1) - \text{Pr}(\hat{\mathbf{y}} = 1 | X_p = 1))
\end{aligned}
\end{equation}
where $\hat{\mathbf{y}}$ are the model decisions, $X_p$ and $X_c$ represent binary markers created from some demographic attribute, $c$ denotes the control group (often whites or males), $p$ indicates a protected group, and $\text{Pr}(\cdot)$ is the operator for conditional probability. ME is a favored DI measure used by the US Consumer Financial Protection Bureau (CFPB), the primary agency charged with regulating fair lending laws at the largest US lending institutions and for various other participants in the consumer financial market (see
\href{https://files.consumerfinance.gov/f/201510_cfpb_supervisory-highlights.pdf}
{\textit{Supervisory Highlights}, Issue 9, Fall 2015}).
Another important DI measure is AIR, more~commonly known as a relative risk ratio in settings outside of regulatory compliance.
\begin{equation}
\label{eq:air}
\begin{aligned}
\text{AIR} \equiv \frac{\text{Pr}(\hat{\mathbf{y}} = 1 | X_p = 1)}{\text{Pr}(\hat{\mathbf{y}} = 1| X_c = 1)}
\end{aligned}
\end{equation}
AIR is equal to the ratio of the proportion of the protected class that receives a favorable outcome and the proportion of the control class that receives a favorable outcome. Statistically significant AIR values below 0.8 can be considered {prima facie} evidence of discrimination. An additional long-standing and pertinent measure of DI is SMD. SMD is often used to assess disparities in continuous features, such~as income differences in employment analyses, or interest rate differences in lending. It originates from work on statistical power, and is more formally known as {Cohen’s d}. SMD is equal to the difference in the average protected class outcome, $\bar{\hat{\mathbf{y}}}_p$, minus the control class outcome, $\bar{\hat{\mathbf{y}}}_c$, divided by a measure of the standard deviation of the population, $\sigma_{\hat{\mathbf{y}}}$. ({{There are several measures of the standard deviation of the score that are typically used}%%% footnote is not allowed in our jounral, we have moved them in the main text, please confirm if you are OK with this.
: \textbf{{1}.} the standard deviation of the population, irrespective of protected class status, \textbf{{2}.} a standard deviation calculated only over the two groups being considered in a particular calculation, or \textbf{{3}.} a pooled standard deviation, using the standard deviations for each of the two groups with weights.}) Cohen defined values of this measure to have {small}, {medium}, and {large} effect sizes if the values exceeded 0.2, 0.5, and 0.8, respectively. %is bold necessary?
\begin{equation}
\label{eq:smd}
\begin{aligned}
\text{SMD} \equiv \frac{\bar{\hat{\mathbf{y}}}_p - \bar{\hat{\mathbf{y}}}_c}{ \sigma_{\hat{\mathbf{y}}}}
\end{aligned}
\end{equation}
The numerator in the SMD is roughly equivalent to ME but adds the standard deviation divisor as a standardizing factor. Because of this standardization factor, SMD allows for a comparison across different types of outcomes, such as inequity in mortgage closing fees or inequities in the interest rates given on certain loans. In this, one may apply definitions in \citet{cohen1988statistical} of {small}, {medium}, and~{large} effect sizes, which represent a measure of {practical significance}, which is described in detail in Appendix \ref{a_sec:prac_v_stat}. Finally, confusion matrix measures in demographic groups, such as accuracy, false~positive rate (FPR), false negative rate (FNR), and their ratios, are also considered as measures of DI in Section~\ref{ssec:dis_mort} and Appendix \ref{a_ssec:dis_sim}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Software Resources}\label{ssec:soft}
Python code to reproduce discussed results is available at: {\url{https://github.com/h2oai/article-information-2019}}. The primary Python packages employed are:
%\href{https://numpy.org/}
{\texttt{{numpy}}} 1.14.5 and
%\href{https://pandas.pydata.org/}
{\texttt{{pandas}}} 0.22.0 for data manipulation,
%\href{https://github.com/h2oai/h2o-3}
{\texttt{{h2o}}} 3.26.0.9,
%\href{https://keras.io/}
{\texttt{{Keras}}} 2.3.1,
%\href{https://github.com/slundberg/shap}
{\texttt{{shap}}} 0.31.0, and
%\href{https://www.tensorflow.org/}
{\texttt{{tensorflow}}} 1.14.0 for modeling, explanation, and discrimination testing, and typically
%\href{https://matplotlib.org/}
{\texttt{{matplotlib}}} 2.2.2 for plotting.
% we have removed the link of text, and keep web link
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Results}\label{sec:res}
Results are laid out for the simulated and mortgage datasets. Accuracy is compared for unconstrained, less interpretable $g^{\text{GBM}}$ and $g^{\text{ANN}}$ models and constrained, more interpretable $g^{\text{MGBM}}$ and $g^{\text{XNN}}$ models. Then, for the $g^{\text{MGBM}}$ and $g^{\text{XNN}}$ models, intrinsic interpretability, post-hoc explanation, and discrimination testing results are explored.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Simulated Data Results}
Fit comparisons between unconstrained and constrained models and XNN interpretability results are discussed in Sections \ref{ssec:c_v_uc_sim} and \ref{ssec:xnn_sim}. As model training and assessment on the simulated data is a rough validation exercise meant to showcase expected results on data with known characteristics, and given that most of the techniques in the proposed workflow are already used widely or have been validated elsewhere, reporting of simulated data results in the main text will focus mostly on fit measures and the more novel $g^\text{XNN}$ interpretability results. The bulk of the post-hoc explanation and discrimination testing results for the simulated data are left to Appendix \ref{a_sec:res_sim}.
\subsubsection{Constrained vs. Unconstrained Model Fit Assessment}\label{ssec:c_v_uc_sim}
Table \ref{tab:sim_acc} presents a variety of fit measures for $g^\text{GBM}$, $g^\text{MGBM}$, $g^\text{ANN}$, and $g^\text{XNN}$ on the simulated test data. $g^\text{XNN}$ exhibited the best performance, but the models exhibited only a fairly small range of fit results. Interpretability and explainability benefits of the constrained models appeared to come at little cost to overall model performance, or in the case of $g^\text{ANN}$ and $g^\text{XNN}$, no cost at all. For the displayed measures, $g^\text{MGBM}$ performed $\sim$2\% worse on average than $g^\text{GBM}$. $g^\text{XNN}$ performed $\sim$0.5\% better on average than $g^\text{XNN}$, and $g^\text{XNN}$ actually showed slightly better fit than $g^\text{ANN}$ across all fit measures except specificity. Fit measures that required a probability cutoff were taken at the best F1 threshold for each model.
\begin{table}[H]%% we have removed the link of text
\caption{Fit measures for $g^\text{GBM}(\mathbf{X})$, $g^\text{MGBM}(\mathbf{X})$, $g^\text{ANN}(\mathbf{X})$, and $g^\text{XNN}(\mathbf{X})$ on the simulated test data. Arrows indicate the direction of improvement for each measure and the best result in each column is displayed in bold font.}
\centering
\begin{tabular}{ccccccccc}
\toprule
\textbf{Model} &
%\href{https://scikit-learn.org/stable/modules/model_evaluation.html#accuracy-score}
{\textbf{{Accuracy}}}~$\uparrow$ &
%\href{https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve}
{\textbf{{AUC}}}~$\uparrow$ &
%\href{https://en.wikipedia.org/wiki/F1_score}
{\textbf{{F1}}}~$\uparrow$ &
%\href{https://scikit-learn.org/stable/modules/model_evaluation.html#log-loss}
{\textbf{{Logloss}}}~$\downarrow$ &
%\href{https://en.wikipedia.org/wiki/Matthews_correlation_coefficient}
{\textbf{{MCC}}}~$\uparrow$ &
%\href{https://en.wikipedia.org/wiki/Root-mean-square_deviation}
{\textbf{{RMSE}}}~$\downarrow$ &
%\href{https://en.wikipedia.org/wiki/Sensitivity_and_specificity}
{\textbf{{Sensitivity}}}~$\uparrow$ &
%\href{https://en.wikipedia.org/wiki/Sensitivity_and_specificity}
{\textbf{{Specificity}}}~$\uparrow$ \\
\midrule
$g^\text{GBM}$ & 0.757 & 0.847 & 0.779 & 0.486 & 0.525 & 0.400 & 0.858 & \textbf{0.657} \\
$g^\text{MGBM}$ & 0.744 & 0.842 & 0.771 & 0.502 & 0.504 & 0.407 & 0.864 & 0.625 \\
$g^\text{ANN}$ & 0.757 & 0.850 & 0.779 & 0.480 & 0.525 & 0.398 & 0.858 & \textbf{0.657} \\
$g^\text{XNN}$ & \textbf{0.758} & \textbf{0.851} & \textbf{0.781} & \textbf{0.479} & \textbf{0.528} & \textbf{0.397} & \textbf{0.867} & 0.648 \\
\bottomrule
\label{tab:sim_acc}
\end{tabular}
\end{table}
\subsubsection{Interpretability Results}\label{ssec:xnn_sim}
For $g^\text{XNN}$, inherent interpretability manifested as plots of sparse $\gamma_k$ output layer weights, $n_k$ subnetwork ridge functions, and sparse $\beta_{j,k}$ weights in the bottom projection layer. Figure \ref{fig:sim_xnn_glob_ridge} provides detailed insights into the structure of $g^\text{XNN}$ (also described in Equation (\ref{eq:xnn})). Figure \ref{fig:sim_xnn_glob_ridge}{a} displays the sparse $\gamma_k$ weights of the output layer, where only $n_k$ subnetworks with $k \in \{1, 4, 7, 8, 9\}$ were associated with large magnitude weights. The $n_k$ subnetwork ridge functions appear in Figure \ref{fig:sim_xnn_glob_ridge}{b} as simplistic but distinctive functional forms. Color-coding between Figure \ref{fig:sim_xnn_glob_ridge}{a},{b} visually reinforces the direct feed-forward relationship between the $n_k$ subnetworks and the $\gamma_k$ weights of the output layer.% we have removed the link of text
% start a new page
\newpage
% change it to landscape
\paperwidth=\pdfpageheight
\paperheight=\pdfpagewidth
\pdfpageheight=\paperheight
\pdfpagewidth=\paperwidth
\newgeometry{layoutwidth=297mm,layoutheight=210 mm, left=2.7cm,right=2.7cm,top=1.8cm,bottom=1.5cm, includehead,includefoot}
\fancyheadoffset[LO,RE]{0cm}
\fancyheadoffset[RO,LE]{0cm}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[H]
\centering
%\includegraphics[width=15cm]{img/sim_xnn_glob_ridge.png}
\includegraphics[width=23cm]{img/sim_xnn_glob_ridge.png}
\caption{Output layer $\gamma_k$ weights, corresponding $n_k$ subnetwork ridge functions, and associated projection layer $\beta_{k,j}$ weights for $g^\text{XNN}$ on the simulated data.}
\label{fig:sim_xnn_glob_ridge}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% change everything back
\newpage
\restoregeometry
\paperwidth=\pdfpageheight
\paperheight=\pdfpagewidth
\pdfpageheight=\paperheight
\pdfpagewidth=\paperwidth
\headwidth=\textwidth
$n_k$ subnetworks were plotted across the output values of their associated $\sum_j\beta_{k,j}x_j$ projection layer hidden units, and color-coding between Figure \ref{fig:sim_xnn_glob_ridge}{b},c link the $\beta_{j,k}$ weights to their $n_k$ subnetworks. Most of the heavily utilized $n_k$ subnetworks had sparse weights in their $\sum_j\beta_{k,j}x_j$ projection layer hidden units. In particular, subnetwork $n_1$ appeared to be almost solely a function of $X_{\text{Friedman}, 3}$ and appeared to exhibit the expected quadratic behavior for $X_{\text{Friedman}, 3}$. Subnetworks $n_7$, $n_8$, and $n_9$ appeaed to be most associated with the globally important $X_{\text{Friedman}, 1}$ and $X_{\text{Friedman}, 2}$ features, likely betraying the effort required for $g^{\text{XNN}}$ to model the nonlinear $\text{sin}()$ function of the $X_{\text{Friedman}, 1}$ and $X_{\text{Friedman}, 2}$ product, and these subnetworks, especially $n_7$ and $n_8$, appeared to display some noticeable sinusoidal characteristics. Subnetwork $n_4$ seemed to be a linear combination of all the original input $X_j$ features, but did weigh the linear $X_{\text{Friedman},4}$ and $X_{\text{Friedman},5}$ terms roughly in the correct two-to-one ratio. As~a whole, Figure \ref{fig:sim_xnn_glob_ridge}a--c exhibited evidence that $g^\text{XNN}$ learned about the signal-generating function in Equation (\ref{eq:sim}) and the displayed information should help practitioners understand which original input $X_j$ features were weighed heavily in each $n_k$ subnetwork, and which $n_k$ subnetworks have a strong influence on $g^\text{XNN}(\mathbf{X})$ output. See Appendix \ref{a_ssec:xnn} for additional details regarding general XNN architecture.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Mortgage Data Results}
Results for the mortgage data are presented in Sections \ref{ssec:c_v_uc_mort}---\ref{ssec:dis_mort} to showcase the example workflow. $g^\text{ANN}$ and $g^\text{XNN}$ outperformed $g^\text{GBM}$ and $g^\text{MGBM}$ on the mortgage data, but as in Section~\ref{ssec:c_v_uc_sim}, the constrained variants of both model architectures did not show large differences in model fit with respect to unconstrained variants. Assuming that in high-stakes applications small fit differences on static test data did not outweigh the need for enhanced model debugging facilitated by high interpretability, only $g^\text{MGBM}$ and $g^\text{XNN}$ interpretability, post-hoc explainability, and discrimination testing results are presented.
\subsubsection{Constrained vs. Unconstrained Model Fit Assessment}\label{ssec:c_v_uc_mort}
Table \ref{tab:mort_acc} shows that $g^\text{ANN}$ and $g^\text{XNN}$ noticeably outperformed $g^\text{GBM}$ and $g^\text{MGBM}$ on the mortgage data for most of the fit measures. This is at least partially due to the preprocessing required to present directly comparable post-hoc explainability results and to use neural networks and TensorFlow, e.g., numerical encoding of categorical features and missing values. This preprocessing appears to hamstring some of the tree-based models' inherent capabilities. $g^\text{GBM}$ models trained on non-encoded data with missing values repeatedly produced receiver operating characteristic area under the curve (AUC) values of $\sim$0.81 (not shown, but available in resources discussed in Section~\ref{ssec:soft}).
\begin{table}[H]%% we have removed the link of text
\caption{Fit measures for $g^\text{GBM}(\mathbf{X})$, $g^\text{MGBM}(\mathbf{X})$, $g^\text{ANN}(\mathbf{X})$, and $g^\text{XNN}(\mathbf{X})$ on the mortgage test data. Arrows indicate the direction of improvement for each measure and the best result in each column is displayed in bold font.}
\centering
\begin{tabular}{ccccccccc}
\toprule
\textbf{Model} &
%\href{https://scikit-learn.org/stable/modules/model_evaluation.html#accuracy-score}
{\textbf{{Accuracy}}}~$\uparrow$ &
%\href{https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve}
{\textbf{{AUC}}}~$\uparrow$ &
%\href{https://en.wikipedia.org/wiki/F1_score}
{\textbf{{F1}}}~$\uparrow$ &
%\href{https://scikit-learn.org/stable/modules/model_evaluation.html#log-loss}
{\textbf{{Logloss}}}~$\downarrow$ &
%\href{https://en.wikipedia.org/wiki/Matthews_correlation_coefficient}
{\textbf{{MCC}}}~$\uparrow$ &
%\href{https://en.wikipedia.org/wiki/Root-mean-square_deviation}
{\textbf{{RMSE}}}~$\downarrow$ &
%\href{https://en.wikipedia.org/wiki/Sensitivity_and_specificity}
{\textbf{{Sensitivity}}}~$\uparrow$ &
%\href{https://en.wikipedia.org/wiki/Sensitivity_and_specificity}
{\textbf{{Specificity}}}~$\uparrow$ \\
\midrule
$g^\text{GBM}$ & 0.795 & 0.828 & 0.376 & 0.252 & 0.314 & 0.276 & 0.634 & 0.813 \\
$g^\text{MGBM}$ & 0.765 & 0.814 & 0.362 & 0.259 & 0.305 & 0.278 & \textbf{0.684} & 0.773 \\
$g^\text{ANN}$ & 0.865 & \textbf{0.871} & \textbf{0.474} & \textbf{0.231} & \textbf{0.418} & \textbf{0.262} & 0.624 & 0.891 \\
$g^\text{XNN}$ & \textbf{0.869} & 0.868 & 0.468 & 0.233 & 0.409 & 0.263 & 0.594 & \textbf{0.898} \\
\bottomrule
\label{tab:mort_acc}
\end{tabular}
\end{table}
Regardless of the fit differences between the two families of models, the difference between the constrained and unconstrained variants within the two types of models is small for the GBMs and smaller for the ANNs, $\sim$3.5\% and $\sim$1\% worse fit respectively, averaged across the measures in Table \ref{tab:mort_acc}.
\subsubsection{Interpretability and Post-hoc Explanation Results}\label{ssec:int_ex_mort}
For $g^\text{MGBM}(\mathbf{X})$, intrinsic interpretability was evaluated with PD and ICE plots of mostly monotonic prediction behavior for several important $X_j$, and post-hoc Shapley explanation analysis was used to create global and local feature importance. Global Shapley feature importance for $g^\text{MGBM}(\mathbf{X})$ on the mortgage test data is reported in Figure \ref{fig:mort_mgbm_glob}. $g^\text{MGBM}$ placed high importance on LTV ratio, perhaps too high, and also weighed DTI ratio, property value, loan amount, and introductory rate period heavily in many of its predictions. Tree SHAP values are reported in the margin space, prior to the application of the logit link function, and the reported numeric values can be interpreted as the mean absolute impact of each $X_j$ on $g^\text{MGBM}(\mathbf{X})$ in the mortgage test data in the $g^\text{MGBM}(\mathbf{X})$ margin space. The~potential over-emphasis of LTV ratio, and the de-emphasis of income, likely an important feature from a business perspective, and the de-emphasis of the encoded no introductory rate period flag feature may also contribute to the decreased performance of $g^\text{MGBM}(\mathbf{X})$ as compared to $g^\text{XNN}(\mathbf{X})$.
\begin{figure}[H]
\centering
%\captionsetup{width=17cm}
%\includegraphics[width=15cm]{img/mort_mgbm_glob.png}
\includegraphics[width=12cm]{img/mort_mgbm_glob.png}
\caption{Global mean absolute Tree Shapley Additive Explanations (SHAP) feature importance for $g^\text{MGBM}(\mathbf{X})$ on the mortgage test data.}
\label{fig:mort_mgbm_glob}
\end{figure}
Domain knowledge was used to positively constrain DTI ratio and LTV ratio and to negatively constrain income and the loan term flag under $g^\text{MGBM}$. The monotonicity constraints for DTI ratio and LTV ratio were confirmed for $g^\text{MGBM}(\mathbf{X})$ on the mortgage test data in Figure \ref{fig:mort_mgbm_glob_pdp_ice}. Both DTI ratio and LTV ratio displayed positive monotonic behavior at all selected percentiles of $g^\text{MGBM}(\mathbf{X})$ for ICE and on average with PD. Because PD curves generally followed the patterns of the ICE curves for both features, it is also likely that no strong interactions were at play for DTI ratio and LTV ratio under $g^\text{MGBM}$. Of course, the monotonicity constraints themselves could have dampened the effects of non-monotonic interactions under $g^\text{MGBM}$, even if they did exist in the training data (e.g., LTV ratio and the no introductory rate period flag, see {Figure} 6). This rigidity could also have played a role in the performance differences between $g^\text{MGBM}(\mathbf{X})$ and $g^\text{XNN}(\mathbf{X})$ in the mortgage data not observed for the simulated data, wherein strong interactions appeared to be between features with the same monotonicity constraints (e.g., $X_{\text{Friedman},1}$ and $X_{\text{Friedman},2}$, see Figure \ref{fig:sim_xnn_glob_ridge}).%Figures should be cited in sequential numerical order. Please confirm if it's invalid citation, if no, please revise the order.
PD and ICE are displayed with a histogram to highlight any sparse regions in an input feature's domain. Because most ML models will always issue a prediction on any instance with a correct schema, it is crucial to consider whether a given model learned enough about an instance to make an accurate prediction. Viewing PD and ICE along with a histogram is a convenient method to visually assess whether a prediction is reasonable and based on sufficient training data. The DTI ratio and LTV ratio do appear to have had sparse regions in their univariate distributions. The monotonicity constraints likely play to the advantage of $g^\text{MGBM}$ in this regard, as $g^\text{MGBM}(\mathbf{X})$ appears to carry reasonable predictions learned from dense domains into the sparse domains of both features.
\begin{figure}[H]
\centering
\includegraphics[width=15cm]{img/mort_mgbm_glob_pdp_ice.png}
\caption{PD, ICE for ten individuals across selected percentiles of $g^\text{MGBM}(\mathbf{X})$, and histograms for the three most important input features of $g^\text{MGBM}$ on the mortgage test data.}
\label{fig:mort_mgbm_glob_pdp_ice}
\end{figure}
Figure \ref{fig:mort_mgbm_glob_pdp_ice} also displays PD and ICE for the unconstrained feature property value. Unlike the DTI ratio and LTV ratio, PD for property value did not always follow the patterns established by ICE curves. While PD showed monotonically increasing prediction behavior on average, apparently influenced by large predictions at extreme $g^{\text{MGBM}}(\mathbf{X})$ percentiles, ICE curves for individuals at the 40{th} percentile of $g^{\text{MGBM}}(\mathbf{X})$ and lower exhibited different prediction behavior with respect to property value. Some individuals at these lower percentiles displayed monotonically decreasing prediction behavior, while others appeared to show fluctuating prediction behavior. Property value was strongly right-skewed, with little data regarding high-value property from which $g^{\text{MGBM}}$ can learn. For the most part, reasonable predictions did appear to be carried from more densely populated regions to more sparsely populated regions. However, prediction fluctuations at lower $g^{\text{MGBM}}(\mathbf{X})$ percentiles were visible, and appeared in a sparse region of property value. This divergence of PD and ICE could be indicative of an interaction affecting property value under $g^{\text{MGBM}}$ \cite{ice_plots}, and analysis by surrogate decision tree did show evidence of numerous potential interactions in lower predictions ranges of $g^{\text{MGBM}}(\mathbf{X})$ \cite{art_and_sci} (not shown, but available in resources discussed in Section~\ref{ssec:soft}). Fluctuations in ICE could also have been caused by overfitting or by leakage of strong non-monotonic signal from important constrained features into the modeled behavior of non-constrained features.
In Figure \ref{fig:mort_mgbm_loc}, local Tree SHAP values are displayed for selected individuals at the 10{th}, 50{th}, and 90{th} percentiles of $g^{\text{MGBM}}(\mathbf{X})$ in the mortgage test data. Each Shapley value in Figure \ref{fig:mort_mgbm_loc} represents the difference in $g^\text{MGBM}(\mathbf{x})$ and the average of $g^\text{MGBM}(\mathbf{X})$ associated with this instance of some input feature $x_j$ \cite{molnar}. Accordingly, the logit of the sum of the Shapley values and the average of $g^\text{MGBM}(\mathbf{X})$ is $g^\text{MGBM}(\mathbf{x})$, the prediction in the probability space for any $\mathbf{x}$.
\begin{figure}[H]
\centering
\includegraphics[width=15cm]{img/mort_mgbm_loc.png}
\caption{Tree SHAP values for three individuals across selected percentiles of $g^\text{MGBM}(\mathbf{X})$ for the mortgage test data.}
\label{fig:mort_mgbm_loc}
\end{figure}
The selected individuals showed an expected progression of mostly negative Shapley values at the 10{th} percentile, a mixture of positive and negative Shapley values at the 50{th} percentile, mostly positive Shapley values at the 90{th} percentile, and with globally important features driving most local model decisions. Deeper significance for Figure \ref{fig:mort_mgbm_loc} lies in the ability of Tree SHAP to accurately and consistently summarize any single $g^\text{MGBM}(\mathbf{x})$ prediction in this manner, which is generally important for enabling logical appeal or override of ML-based decisions, and is specifically important in the context of lending, where applicable regulations often require lenders to provide consumer-specific reasons for denying credit to an individual. In the US, applicable regulations are typically ECOA and FCRA, and the consumer-specific reasons are commonly known as \textit{adverse actions codes}.
Figure \ref{fig:mort_xnn_glob} displays global feature importance for $g^\text{XNN}(\mathbf{X})$ on the mortgage test data. Deep SHAP values are reported in the probability space, after the application of the logit link function. They are also calculated from the projection layer of $g^\text{XNN}$. Thus, the Deep SHAP values in Figure \ref{fig:mort_xnn_glob} are the estimated average absolute impact of each input, $X_j$, in the projection layer and probability space of $g^\text{XNN}(\mathbf{X})$ for the mortgage test data. $g^\text{XNN}$ distributes importance more evenly across business drivers and puts stronger emphasis on the no introductory rate period flag feature than does $g^\text{MGBM}$. Like $g^\text{MGBM}$, $g^\text{XNN}$ puts little emphasis on the other flag features. Unlike $g^\text{MGBM}$, $g^\text{XNN}$ assigned higher importance to property value, loan amount, and income, and lower importance on LTV ratio and DTI ratio.
\begin{figure}[H]
\centering
%\includegraphics[width=15cm]{img/mort_xnn_glob.png}
\includegraphics[width=12cm]{img/mort_xnn_glob.png}
\caption{Global mean absolute Deep SHAP feature importance for $g^\text{XNN}(\mathbf{X})$ on the mortgage test data.}
\label{fig:mort_xnn_glob}
\end{figure}
The capability of $g^\text{XNN}$ to model nonlinear phenomenon and high-degree interactions, and to do so in an interpretable manner, is on display in Figure \ref{fig:mort_xnn_glob_ridge}. Figure \ref{fig:mort_xnn_glob_ridge}{a} presents the sparse $\gamma_k$ weights of the $g^\text{XNN}$ output layer in which the $n_k$ subnetworks with $k \in \{0,1,2,3,5,8,9\}$ had large magnitude weights and $n_k$ subnetworks, $k \in \{4,6,7\}$, had small or near-zero magnitude weights. Distinctive ridge functions that fed into those large magnitude $\gamma_k$ weights are highlighted in Figure \ref{fig:mort_xnn_glob_ridge}{b} and color-coded to pair with their corresponding $\gamma_k$ weight. As in the Section~\ref{ssec:xnn_sim}, $n_k$ ridge function plots varied with the output of the corresponding projection layer $\sum_j\beta_{k,j}x_j$ hidden unit, with weights displayed in matching colors in Figure \ref{fig:mort_xnn_glob_ridge}{c}. In both the simulated and mortgage data, $n_k$ ridge functions appeared to be elementary functional forms that the output layer learned to combine to generate accurate predictions, reminiscent of basis functions for the modeled space. Figure \ref{fig:mort_xnn_glob_ridge}{c} displays the sparse $\beta_{j,k}$ weights of the projection layer $\sum_j\beta_{k,j}x_j$ hidden units that were associated with each $n_k$ subnetwork ridge function. For instance, subnetwork $n_3$ was influenced by large weights for LTV ratio, no introductory rate period flag, and introductory rate period, whereas subnetwork $n_9$ was nearly completely dominated by the weight for income. See Appendix \ref{a_ssec:xnn} for details regarding general XNN architecture.
% start a new page
\newpage
% change it to landscape
\paperwidth=\pdfpageheight
\paperheight=\pdfpagewidth
\pdfpageheight=\paperheight
\pdfpagewidth=\paperwidth
\newgeometry{layoutwidth=297mm,layoutheight=210 mm, left=2.7cm,right=2.7cm,top=1.8cm,bottom=1.5cm, includehead,includefoot}
\fancyheadoffset[LO,RE]{0cm}
\fancyheadoffset[RO,LE]{0cm}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{figure}[H]
\includegraphics[width=23cm]{img/mort_xnn_glob_ridge.png}
\caption{Output layer $\gamma_k$ weights, corresponding $n_k$ ridge functions, and associated projection layer $\beta_{k,j}$ weights for $g^\text{XNN}$ on the mortgage data.}
\label{fig:mort_xnn_glob_ridge}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% change everything back
\newpage
\restoregeometry
\paperwidth=\pdfpageheight
\paperheight=\pdfpagewidth
\pdfpageheight=\paperheight
\pdfpagewidth=\paperwidth
\headwidth=\textwidth
To complement the global interpretability of $g^\text{XNN}$, Figure \ref{fig:mort_xnn_loc} displays local Shapley values for selected individuals, estimated from the projection layer using Deep SHAP in the $g^\text{XNN}$ probability space. Similar to Tree SHAP, local Deep SHAP values should sum to $g^\text{XNN}(\mathbf{x})$. While the Shapley values appeared to follow the roughly increasing pattern established in Figures \ref{fig:sim_mgbm_loc}, \ref{fig:sim_xnn_loc}, and \ref{fig:mort_mgbm_loc}, their true value was their ability to be calculated for any $g^\text{XNN}(\mathbf{x})$ prediction, as a means to summarize model reasoning and allow for appeal and override of specific ML-based decisions.
\begin{figure}[H]
\centering
\includegraphics[width=15cm]{img/mort_xnn_loc.png}
\caption{Deep SHAP values for three individuals across selected percentiles of $g^\text{XNN}(\mathbf{X})$ on the mortgage test data.}
\label{fig:mort_xnn_loc}
\end{figure}
\subsubsection{Discrimination Testing Results}\label{ssec:dis_mort}
Table \ref{tab:mort_disc}{a,b} show the results of the discrimination tests using the mortgage data for two sets of class-control groups: blacks as compared to whites, and females as compared to males. As with the simulated data in Table \ref{tab:sim_disc}, several measures of disparities are shown, with the SMDs calculated using the probabilities from $g^{\text{MGBM}}(\mathbf{X})$ and $g^{\text{XNN}}(\mathbf{X})$, and the accuracy, FPRs, and FPR ratios, MEs, and AIRs calculated using a binary outcome based on a cutoff of 0.20 (anyone with probabilities of 0.2 or less receives the favorable outcome; see Appendix \ref{a_sec:cut} for comments pertaining to discrimination testing and cutoff selection).
Since $g^{\text{MGBM}}$ and $g^{\text{XNN}}$ were predicting the likelihood of receiving a high-priced loan, $g^{\text{MGBM}}$ and $g^{\text{XNN}}$ assume that a lower score was favorable. Thus, one might consider FPR ratios as a measure of the class-control disparities. FPR ratios were higher under $g^{\text{XNN}}$ than $g^{\text{MGBM}}$ (2.45 vs. 2.10) in Table \ref{tab:mort_disc}{b}, but overall FPRs were lower for blacks under $g^{\text{XNN}}$ (0.295 vs. 0.315) in Table \ref{tab:mort_disc}{a}. This is the same pattern seen in the simulated data results in Appendix \ref{a_ssec:dis_sim}, leading to the question of whether a fairness goal should not only consider class-control relative rates, but also intra-class improvements in the chosen fairness measure. Similar results were found for the female-male comparison, but the relative rates are less stark: 1.15 for $g^{\text{MGBM}}(\mathbf{X})$ and 1.21 for $g^{\text{XNN}}(\mathbf{X})$.
%\begin{table}[H]
% \hskip-10pt\begin{subtable}{0.45\textwidth}
% \captionsetup{width=7cm}
% \caption{Group size, accuracy, and FPR for $g^\text{MGBM}(\mathbf{X})$ and $g^\text{XNN}(\mathbf{X})$ on the mortgage test data.}
% \centering
% \resizebox{\columnwidth}{!}{%
% \begin{tabular}{ccccc}
% \toprule
% \textbf{Class} & $\mathbf{N}$ & \textbf{Model} & \textbf{Accuracy}$\uparrow$ & \textbf{FPR}$\downarrow$ \\
% \midrule
% Black & 2,608 & \begin{tabular}{@{}c@{}} $g^\text{MGBM}$ \\ $g^\text{XNN}$\end{tabular} & \begin{tabular}{@{}c@{}} 0.654 \\ 0.702\end{tabular} & \begin{tabular}{@{}c@{}} 0.315 \\ 0.295\end{tabular} \\
% \midrule
% White & 28,361 & \begin{tabular}{@{}c@{}} $g^\text{MGBM}$ \\ $g^\text{XNN}$\end{tabular} & \begin{tabular}{@{}c@{}} 0.817 \\ 0.857\end{tabular} & \begin{tabular}{@{}c@{}} 0.150 \\ 0.120\end{tabular}\\
% \midrule
% Female & 8,301 & \begin{tabular}{@{}c@{}} $g^\text{MGBM}$ \\ $g^\text{XNN}$\end{tabular} & \begin{tabular}{@{}c@{}} 0.768 \\ 0.822\end{tabular} & \begin{tabular}{@{}c@{}} 0.208 \\ 0.158\end{tabular}\\
% \midrule
% Male & 13,166 & \begin{tabular}{@{}c@{}} $g^\text{MGBM}$ \\ $g^\text{XNN}$\end{tabular} & \begin{tabular}{@{}c@{}} 0.785 \\ 0.847\end{tabular} & \begin{tabular}{@{}c@{}} 0.182 \\ 0.131\end{tabular}\\
% \bottomrule
% \label{tab:mort_n_acc}
% \end{tabular}%
% }
% \end{subtable}
% \hspace{10pt}
% \begin{subtable}{0.45\textwidth}
% \vspace{-14pt}
% \captionsetup{width=10cm}
% \caption{%
% \leftskip=45pt
% AIR, ME, SMD, and FPR ratio for $g^\text{MGBM}(\mathbf{X})$ and $g^\text{XNN}(\mathbf{X})$ on the mortgage test data.}
% \centering
% \scalebox{0.95}{
% \begin{tabular}{ccccccc}
% \toprule
% \textbf{Model} & \begin{tabular}[x]{@{}c@{}}\textbf{Protected}\\\textbf{Class}\end{tabular} & \begin{tabular}[x]{@{}c@{}}\textbf{Control}\\\textbf{Class}\end{tabular} & \textbf{AIR}$\uparrow$ & \textbf{ME}$\downarrow$ & \textbf{SMD}$\downarrow$ & \begin{tabular}[x]{@{}c@{}}\textbf{FPR}\\\textbf{Ratio}\end{tabular}$\downarrow$\\
% \midrule
% $g^\text{MGBM}$ & \begin{tabular}{@{}c@{}} Black \\ Female\end{tabular} & \begin{tabular}{@{}c@{}} White \\ Male\end{tabular} & \begin{tabular}{@{}c@{}} 0.776 \\ 0.948\end{tabular} & \begin{tabular}{@{}c@{}} 18.3\% \\ 4.1\%\end{tabular} & \begin{tabular}{@{}c@{}} 0.628 \\ 0.084\end{tabular} & \begin{tabular}{@{}c@{}} 2.10 \\ 1.15\end{tabular} \\
% \midrule
% $g^\text{XNN}$ & \begin{tabular}{@{}c@{}} Black \\ Female\end{tabular} & \begin{tabular}{@{}c@{}} White \\ Male\end{tabular} & \begin{tabular}{@{}c@{}} 0.743 \\ 0.955\end{tabular} & \begin{tabular}{@{}c@{}} 21.4\% \\ 3.6\%\end{tabular} & \begin{tabular}{@{}c@{}} 0.621 \\ 0.105\end{tabular} & \begin{tabular}{@{}c@{}} 2.45 \\ 1.21\end{tabular} \\
% \bottomrule
% \label{tab:mort_disc_detail}
% \end{tabular}
% }
% \end{subtable}
%\captionsetup{width=10cm}
%\caption{Discrimination measures for the mortgage test data. Arrows indicate the direction of improvement for each measure.}
%\label{tab:mort_disc}
%\end{table}
\begin{table}[H]
\centering
\caption{Discrimination measures for the mortgage test data. Arrows indicate the direction of improvement for each measure.}
\label{tab:mort_disc}
\begin{tabular}{ccccc}
\toprule
\multicolumn{5}{l}{\begin{tabular}{@{}c@{}} (\textbf{a}) Group size, accuracy, and FPR for $g^\text{MGBM}(\mathbf{X})$\\ and $g^\text{XNN}(\mathbf{X})$ on the mortgage test data. \end{tabular}}\\
\midrule
\textbf{Class} & $\mathbf{N}$ & \textbf{Model} & \textbf{Accuracy}$\uparrow$ & \textbf{FPR}$\downarrow$ \\
\midrule
Black & 2608 & \begin{tabular}{@{}c@{}} $g^\text{MGBM}$ \\ $g^\text{XNN}$\end{tabular} & \begin{tabular}{@{}c@{}} 0.654 \\ 0.702\end{tabular} & \begin{tabular}{@{}c@{}} 0.315 \\ 0.295\end{tabular} \\
\midrule
White & 28,361 & \begin{tabular}{@{}c@{}} $g^\text{MGBM}$ \\ $g^\text{XNN}$\end{tabular} & \begin{tabular}{@{}c@{}} 0.817 \\ 0.857\end{tabular} & \begin{tabular}{@{}c@{}} 0.150 \\ 0.120\end{tabular}\\
\midrule
Female & 8301 & \begin{tabular}{@{}c@{}} $g^\text{MGBM}$ \\ $g^\text{XNN}$\end{tabular} & \begin{tabular}{@{}c@{}} 0.768 \\ 0.822\end{tabular} & \begin{tabular}{@{}c@{}} 0.208 \\ 0.158\end{tabular}\\
\midrule
Male & 13,166 & \begin{tabular}{@{}c@{}} $g^\text{MGBM}$ \\ $g^\text{XNN}$\end{tabular} & \begin{tabular}{@{}c@{}} 0.785 \\ 0.847\end{tabular} & \begin{tabular}{@{}c@{}} 0.182 \\ 0.131\end{tabular}\\
\bottomrule
\label{tab:mort_n_acc}
\end{tabular}\\
\begin{tabular}{ccccccc}
\toprule
\multicolumn{7}{l}{\begin{tabular}{@{}c@{}} (\textbf{b}) AIR, ME, SMD, and FPR ratio for $g^\text{MGBM}(\mathbf{X})$ and $g^\text{XNN}(\mathbf{X})$\\ on the mortgage test data. \end{tabular}} \\
\midrule
\textbf{Model} & \begin{tabular}[x]{@{}c@{}}\textbf{Protected}\\\textbf{Class}\end{tabular} & \begin{tabular}[x]{@{}c@{}}\textbf{Control}\\\textbf{Class}\end{tabular} & \textbf{AIR}$\uparrow$ & \textbf{ME}$\downarrow$ & \textbf{SMD}$\downarrow$ & \begin{tabular}[x]{@{}c@{}}\textbf{FPR}\\\textbf{Ratio}\end{tabular}$\downarrow$\\
\midrule
$g^\text{MGBM}$ & \begin{tabular}{@{}c@{}} Black \\ Female\end{tabular} & \begin{tabular}{@{}c@{}} White \\ Male\end{tabular} & \begin{tabular}{@{}c@{}} 0.776 \\ 0.948\end{tabular} & \begin{tabular}{@{}c@{}} 18.3\% \\ 4.1\%\end{tabular} & \begin{tabular}{@{}c@{}} 0.628 \\ 0.084\end{tabular} & \begin{tabular}{@{}c@{}} 2.10 \\ 1.15\end{tabular} \\
\midrule
$g^\text{XNN}$ & \begin{tabular}{@{}c@{}} Black \\ Female\end{tabular} & \begin{tabular}{@{}c@{}} White \\ Male\end{tabular} & \begin{tabular}{@{}c@{}} 0.743 \\ 0.955\end{tabular} & \begin{tabular}{@{}c@{}} 21.4\% \\ 3.6\%\end{tabular} & \begin{tabular}{@{}c@{}} 0.621 \\ 0.105\end{tabular} & \begin{tabular}{@{}c@{}} 2.45 \\ 1.21\end{tabular} \\
\bottomrule
\label{tab:mort_disc_detail}
\end{tabular}
\end{table}
Both ME and AIR showed higher disparities for blacks under $g^{\text{XNN}}$ than $g^{\text{MGBM}}$. Blacks receive high-priced loans 21.4\% more frequently using $g^{\text{XNN}}$ vs. 18.3\% for $g^{\text{MGBM}}$. Both $g^{\text{MGBM}}$ and $g^{\text{XNN}}$ showed AIRs that were statistically significantly below parity (not shown, but available in resources discussed in Section~\ref{ssec:soft}), and which were also below the EEOC’s 0.80 threshold. This would typically indicate need for further review to determine the cause and validity of these disparities, and a few relevant remediation techniques for such discovered discrimination are discussed in Section~\ref{ssec:disc_rem}. On the other hand, women improved under $g^{\text{XNN}}$ vs. $g^{\text{MGBM}}$ (MEs of 3.6\% vs. 4.1\%; AIRs of 0.955 vs. 0.948). The AIRs, while statistically significantly below parity, were well above the EEOC’s threshold of 0.80. In most situations, the values of these measures alone would not likely flag a model for further review. Black SMDs for $g^{\text{XNN}}(\mathbf{X})$ and $g^{\text{MGBM}}(\mathbf{X})$ were similar: 0.621 and 0.628, respectively. These exceeded Cohen’s guidelines of 0.5 for a medium effect size and would likely trigger further review. Female SMDs were well below Cohen’s definition of small effect size: 0.105 and 0.084 for $g^{\text{XNN}}(\mathbf{X})$ and $g^{\text{MGBM}}(\mathbf{X})$, respectively. Similar to results for female AIR, these values alone are unlikely to prompt further review.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Discussion}\label{sec:disc}
\vspace{-6pt}
\subsection{The Burgeoning Python Ecosystem for Responsible Machine Learning}
Figure \ref{fig:blueprint} displays a holistic approach to ML model training, assessment, and deployment meant to decrease discrimination, inaccuracy, and privacy and security risks for high-stakes, human-centered, or regulated ML applications. (See \href{https://github.com/jphall663/hc_ml}{Toward Responsible Machine Learning} for details regarding Figure~\ref{fig:blueprint}.) While all the methods mentioned in Figure \ref{fig:blueprint} play an important role in increasing human trust and understanding of ML, a few pertinent references and Python resources are highlighted below as further reading to augment this this text's focus on certain interpretable models, post-hoc explanation, and discrimination testing techniques.
\begin{figure}[H]
\centering
\includegraphics[width=15cm]{img/blueprint.png}
\caption{An example responsible ML workflow in which interpretable models, post-hoc explanations, discrimination testing and remediation techniques, among several other processes, can create an understandable and trustworthy ML system for high-stakes, human-centered, or regulated applications.}
\label{fig:blueprint}
\end{figure}
Any discussion of interpretable ML models would be incomplete without references to the seminal work of the Rudin group at Duke University and EBMs, or GA\textsuperscript{2}Ms, pioneered by researchers at Microsoft and Cornell \cite{ga2m,osdt,this_looks_like_that}. In keeping with a major theme of this manuscript, models from these leading researchers and several other kinds of interpretable ML models are now available as open source Python packages. Among several types of currently available interpretable models, practitioners can now use Python to evaluate EBM in the % we have removed the link of text
%\href{https://github.com/interpretml/interpret}
{\texttt{{interpret}}} package, optimal sparse decision trees, GAMs in the
%\href{https://github.com/dswah/pyGAM}
{\texttt{{pyGAM}}} package, a variant of Friedman's RuleFit in the
%\href{https://github.com/scikit-learn-contrib/skope-rules}
{\texttt{{skope-rules}}} package, monotonic calibrated interpolated lookup tables in
%\href{https://github.com/tensorflow/lattice}
{\texttt{{tensorflow/lattice}}}, and \textit{this looks like that} interpretable deep learning \cite{osdt,rulefit,lattice,this_looks_like_that} (see
\href{https://github.com/xiyanghu/OSDT}{Optimal Sparse Decision Trees},
\href{https://github.com/cfchen-duke/ProtoPNet}{ProtoPNet (\textit{this looks like that})}). Additional, relevant references and Python functionality include:
\begin{itemize}[leftmargin=*,labelsep=5.8mm]%is bold necessary?
\item \textbf{{Exploratory data analysis (EDA)}}: \texttt{H2OAggregatorEstimator} in
%\href{http://docs.h2o.ai/h2o/latest-stable/h2o-py/docs/modeling.html#h2oaggregatorestimator}
{\texttt{{h2o}}} \cite{wilkinson2018visualizing}.
\item \textbf{Sparse feature extraction}: \texttt{H2OGeneralizedLowRankEstimator} in
%\href{http://docs.h2o.ai/h2o/latest-stable/h2o-py/docs/modeling.html#h2ogeneralizedlowrankestimator}
{\texttt{{h2o}}} \cite{udell2016generalized}.
\item \textbf{Preprocessing and models for privacy}:
%\href{https://github.com/IBM/differential-privacy-library}
{\texttt{diffprivlib}},
%\href{https://github.com/tensorflow/privacy/}
{\texttt{{tensorflow/privacy}}} \cite{diffprivlib,ji2014differential,pate,abadi2016deep}.
\item \textbf{Causal inference and probabilistic programming}:
%\href{https://github.com/microsoft/dowhy}
{{dowhy}},
%\href{https://docs.pymc.io/}
{PyMC3} \cite{book_of_why}.
\item \textbf{Post-hoc explanation}: structured data explanations with
%\href{https://github.com/SeldonIO/alibi}
{\texttt{{alibi}}} and
%\href{https://github.com/SauceCat/PDPbox}
{\texttt{{PDPbox}}}, image classification explanations with
%\href{https://github.com/marcoancona/DeepExplain}
{\texttt{{DeepExplain}}}, and natural language explanations with
%\href{https://github.com/allenai/allennlp}
{\texttt{{allennlp}}} \cite{wachter2017counterfactual,grad_attr,wallace2019allennlp}.
\item \textbf{Discrimination testing}:
%\href{https://github.com/dssg/aequitas}
{\texttt{aequitas}},
%\href{https://github.com/LASER-UMASS/Themis}
{\texttt{Themis}}.
\item \textbf{Discrimination remediation}: Reweighing, adversarial de-biasing, learning fair representations, and reject option classification with
%\href{http://aif360.mybluemix.net/}
{\texttt{{AIF}360}} \cite{kamiran2012data,zhang2018mitigating,lfr,kamiran2012decision}.
\item \textbf{Model debugging}:
%\href{https://github.com/bethgelab/foolbox}
{\texttt{{foolbox}}},
%\href{https://github.com/SALib/SALib}
{\texttt{{SALib}}},
%\href{https://github.com/tensorflow/cleverhans}
{\texttt{{tensorflow/cleverhans}}}, and
%\href{https://github.com/tensorflow/model-analysis}
{\texttt{{tensorflow/model- analysis}}} \cite{rauber2017foolbox,papernot2018cleverhans,modeltracker,papernot2018marauder}.
\item \textbf{Model documentation}: models cards \cite{model_cards}, e.g.,
\href{https://github.com/openai/gpt-2/blob/master/model_card.md}{GPT-2 model card},
\href{https://modelcards.withgoogle.com/object-detection}{Object Detection model card}.
\end{itemize}
See
\href{https://github.com/jphall663/awesome-machine-learning-interpretability}
{Awesome Machine Learning Interpretability} for a longer, community-curated metalist of related software packages and resources.
\subsection{Appeal and Override of Automated Decisions}
Interpretable models and post-hoc explanations can play an important role in increasing transparency into model mechanisms and predictions. As seen in Section \ref{sec:res}, interpretable models often enable users to enforce domain knowledge-based constraints on model behavior, to ensure that models obey reasonable expectations, and to gain data-derived insights into the modeled problem domain. Post-hoc explanations generally help describe and summarize mechanisms and decisions, potentially yielding an even clearer understanding of ML models. Together they can allow for human learning from ML, certain types of regulatory compliance, and crucially, human appeal or override of automated model decisions \cite{art_and_sci}. Interpretable models and post-hoc explanations are likely good candidates for ML uses cases under the FCRA, ECOA, GDPR and other regulations that may require explanations of model decisions, and they are already used in the financial services industry today for model validation and other purposes.
(For examples uses in financial services, see
\href{https://ww2.amstat.org/meetings/jsm/2019/onlineprogram/AbstractDetails.cfm?abstractid=303053}
{Deep Insights into Explainability and Interpretability of Machine Learning Algorithms and Applications to Risk Management}. Also note that many non-consistent explanation methods can result in drastically different global and local feature importance values across different models trained on the same data or even for refreshing a model with augmented training data \cite{molnar}. Consistency and accuracy guarantees are perhaps a factor in the growing momentum behind Shapley values as a candidate technique for generating consumer-specific adverse action notices for explaining and appealing automated ML-based decisions in highly-regulated settings, such as credit lending \cite{bracke2019machine}.) In general, transparency in ML also facilitates additional responsible AI processes such as model debugging, model documentation, and logical appeal and override processes, some of which may also be required by applicable regulations (e.g., US Federal Reserve Bank SR 11-7: Guidance on Model Risk Management).
Among these, providing persons affected by a model with the opportunity to appeal ML-based decisions may deserve the most attention. ML models are often wrong (``All models are wrong, but some are useful.''---George Box, Statistician (1919--2013))
and appealing black-box decisions can be difficult (e.g.,
\href{https://www.nytimes.com/2017/06/13/opinion/how-computers-are-harming-criminal-justice.html}
{\textit{When a Computer Program Keeps You in Jail}}). For high-stakes, human-centered, or regulated applications that are trusted with mission- or life-critical decisions, the ability to logically appeal or override inevitable wrong decisions is not only a possible prerequisite for compliance, but also a failsafe procedure for those affected by ML decisions.
\subsection{Discrimination Testing and Remediation in Practice}\label{ssec:disc_rem}
A significant body of research has emerged around exploring and fixing ML discrimination \cite{friedler2019comparative}. Methods can be broadly placed into two groups: more traditional methods that mitigate discrimination by searching across possible algorithmic and feature specifications, and many approaches that have been developed in the last 5--7 years that alter the training algorithm, preprocess training data, or~post-process predictions in order to diminish class-control correlations or dependencies. Whether these more recent methods are suitable for a particular use case depends on the legal environment where a model is deployed and on the use case itself. For comments on why some recent techniques could result in regulatory non-compliance in certain scenarios, see Appendix \ref{a_sec:comp}.
Of the newer class of fairness enhancing interventions, within-algorithm discrimination mitigation techniques that do not use class information may be more likely to be acceptable in highly regulated settings today. These techniques often incorporate a loss function where more discriminatory paths or weights are penalized and only used by the model if improvements in fit overcome some penalty. (The relative level of fit-to-discrimination penalty is usually determined via hyperparameter.) Other mitigation strategies that only alter hyperparameters or algorithm choice are also likely to be acceptable. Traditional feature selection techniques (e.g., those used in linear models and decision trees) are also likely to continue to be accepted in regulatory environments. For further discussion of techniques that can mitigate DI in US financial services, see \citet{schmidt2019introduction}.
Regardless of the methodology chosen to minimize disparities, advances in computing have enhanced the ability to search for less discriminatory models. Prior to these advances, only a small number of alternative algorithms could be tested for lower levels of disparity without causing infeasible delays in model implementation. Now, large numbers of models can be quickly tested for lower discrimination and better predictive quality. An additional opportunity arises as a result of ML itself: the well-known Rashomon effect, or the multiplicity of good ML models for most datasets. It is now feasible to train more models, find more good models, and test more models for discrimination, and among all those tested models, there are likely to be some with high predictive performance and low discrimination.
\subsection{Intersectional and Non-static Risks in Machine Learning}
The often black-box nature of ML, the perpetuation or exacerbation of discrimination by ML, or~the privacy harms and security vulnerabilities inherent in ML are each serious and difficult problems on their own. However, evidence is mounting that these harms can also manifest as complex intersectional challenges, e.g., the {fairwashing} or {scaffolding} of biased models with ML explanations, the privacy harms of ML explanations, or the adversarial poisoning of ML models to become discriminatory \cite{shokri2019privacy,fair_washing,scaffolding} (e.g., \href{https://www.theguardian.com/technology/2016/mar/24/tay-microsofts-ai-chatbot-gets-a-crash-course-in-racism-from-twitter}
{\textit{Tay, Microsoft's AI chatbot, gets a crash course in racism from Twitter}}). (While the focus of this paper is not ML security, proposed best-practices from that field do point to transparency of ML systems as a mitigating factor for some ML attacks and hacks \cite{papernot2018marauder}. High system complexity is sometimes considered a mitigating influence as well \cite{hoare19811980}. This~is sometimes known as the {transparency paradox} in data privacy and security, and it likely applies to ML security as well, especially in the context of interpretable ML models and post-hoc explanation (see \href{https://hbr.org/2019/12/the-ai-transparency-paradox}{\textit{The AI Transparency Paradox}}).) Practitioners should of course consider the discussed interpretable modeling, post-hoc explanation, and~discrimination testing approaches as at least partial remedies to the black-box and discrimination issues in ML. However, they should also consider that explanations can ease model stealing, data~extraction, and membership inference attacks, and that explanations can mask ML discrimination. Additionally, high-stakes, human-centered, or regulated ML systems should generally be built and tested with robustness to adversarial attacks as a primary design consideration, and~specifically to prevent ML models from being poisoned or otherwise altered to become discriminatory. Accuracy, discrimination, and security characteristics of a system can change over time as well. Simply testing for these problems at training time, as presented in Section \ref{sec:res}, is~not adequate for high-stakes, human-centered, or regulated ML systems. Accuracy, discrimination, and~security should be monitored in real-time and over time, as long as a model is deployed. % we have removed the link of text
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}\label{sec:con}
This text puts forward results on simulated data to provide some validation of constrained ML models, post-hoc explanation techniques, and discrimination testing methods. These same modeling, explanation, and discrimination testing approaches are then applied to more realistic mortgage data to provide an example of a responsible ML workflow for high-stakes, human-centered, or regulated ML applications. The discussed methodologies are solid steps toward interpretability, explanation, and minimal discrimination for ML decisions, which should ultimately enable increased fairness and logical appeal processes for ML decision subjects. Of course, there is more to the responsible practice of ML than interpretable models, post-hoc explanation, and discrimination testing, even from a technology perspective, and Section \ref{sec:disc} also points out numerous additional references and open source Python software assets that are available to researchers and practitioners today to increase human trust and understanding in ML systems. While the complex and messy problems of racism, sexism, privacy violations, and cyber crime can probably never be solved by technology alone, this work and many others illustrate numerous ways for ML practitioners to mitigate such risks.
\vspace{+6pt}
%\AuthorNames{Navdeep Gill, Patrick Hall, Kim Montgomery and Nicholas Schmidt}
%%%%%%%%
\authorcontributions{N.G., data cleaning; GBM and MGBM, assessment and results; P.H., primary author; K.M., ANN, and XNN, implementation, assessment, and results; N.S., secondary author, data simulation and collection, and discrimination testing. All authors have read and agreed to the published version of the manuscript.}%For research articles with several authors, a short paragraph specifying their individual contributions must be provided. The following statements should be used ``conceptualization, X.X. and Y.Y.; methodology, X.X.; software, X.X.; validation, X.X., Y.Y. and Z.Z.; formal analysis, X.X.; investigation, X.X.; resources, X.X.; data curation, X.X.; writing--original draft preparation, X.X.; writing--review and editing, X.X.; visualization, X.X.; supervision, X.X.; project administration, X.X.; funding acquisition, Y.Y.'', please turn to the \href{http://img.mdpi.org/data/contributor-role-instruction.pdf}{CRediT taxonomy} for the term explanation. Authorship must be limited to those who have contributed substantially to the work reported.}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\funding{This work received no external funding.}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgments{Wen Phan for work in formalizing notation. Sue Shay for editing. Andrew Burt for ideas around the transparency paradox.}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\conflictsofinterest{{The authors declare no conflict of interest}. %%Please confirm.
XNN was first made public by the corporate model validation team at Wells Fargo bank. Wells Fargo is a customer of, and investor in, H2O.ai and a client of BLDS, LLC. However, communications regarding XNN between Wells Fargo and Patrick Hall at H2O.ai have been extremely limited prior to and during the drafting of this manuscript. Moreover, Wells Fargo exerted absolutely no editorial control over the text or results herein.}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% optional
%\abbreviations{The following abbreviations are used in this text}
\abbreviations{The following abbreviations are used in this manuscript:\\
\noindent
\begin{tabular}{@{}ll}
AI & artificial intelligence\\
AIR & adverse impact ratio\\
ALE & accumulated local effect\\
ANN & artificial neural network\\ % ----------------------------------------------------------------------- x
APR & annual percentage rate\\ % -------------------------------------------------------------------------- x
AUC & area under the curve\\ % ---------------------------------------------------------------------------- x
CFPB & Consumer Financial Protection Bureau\\% ----------------------------------------------------------- x
DI & disparate impact\\ % --------------------------------------------------------------------------------- x
DT & disparate treatment\\% ------------------------------------------------------------------------------ x
DTI & debt to income\\ % ---------------------------------------------------------------------------------- x
\multirow{2}{*}{EBM or GA$\textsuperscript{2}$M} & explainable boosting machine, i.e., variants GAMs that consider two-way interactions and \\ &may incorporate boosting into training\\%
EEOC & Equal Employment Opportunity Commission\\%
ECOA & Equal Credit Opportunity Act\\% -------------------------------------------------------------------- x
EDA & exploratory data analysis\\ % ----------------------------------------------------------------------- x
EU & European Union\\ % ----------------------------------------------------------------------------------- x
FCRA & Fair Credit Reporting Act\\ % ---------------------------------------------------------------------- x
FNR & false negative rate\\ % ----------------------------------------------------------------------------- x
FPR & false positive rate\\ % ----------------------------------------------------------------------------- x
GAM & generalized additive model\\ % ---------------------------------------------------------------------- x
GBM & gradient boosting machine\\ % ----------------------------------------------------------------------- x
GDPR & General Data Protection Regulation\\% -------------------------------------------------------------- x
%MGBM & monotonic gradient boosting machine\\
%ML & machine learning\\ % --------------------------------------------------------------------------------- x
%PD & partial dependence\\ % ------------------------------------------------------------------------------- x
%RMSE & root mean square error\\ % ------------------------------------------------------------------------- x
%SGD & stochastic gradient descent\\% --------------------------------------------------------------------- x
%SHAP & Shapley Additive Explanation\\ % ------------------------------------------------------------------- x
%SMD & standardized mean difference\\ % --------------------------------------------------------------------- x
%SR & supervision and regulation\\ % ----------------------------------------------------------------------- x
%US & United States\\
%XNN & explainable neural network
\end{tabular}
\noindent
\begin{tabular}{@{}ll}
HMDA & Home Mortgage Disclosure Act\\% ------------------------------------------------------------------- x
ICE & individual conditional expectation\\% -------------------------------------------------------------- x
LTV & loan to value\\ % ----------------------------------------------------------------------------------- x
MCC & Matthews correlation coefficient\\% ---------------------------------------------------------------- x
ME & marginal effect\\ % ---------------------------------------------------------------------------------- x
MGBM & monotonic gradient boosting machine\\
ML & machine learning\\ % --------------------------------------------------------------------------------- x
PD & partial dependence\\ % ------------------------------------------------------------------------------- x
RMSE & root mean square error\\ % ------------------------------------------------------------------------- x
SGD & stochastic gradient descent\\% --------------------------------------------------------------------- x
SHAP & Shapley Additive Explanation\\ % ------------------------------------------------------------------- x
SMD & standardized mean difference\\ % --------------------------------------------------------------------- x
SR & supervision and regulation\\ % ----------------------------------------------------------------------- x
US & United States\\
XNN & explainable neural network
\end{tabular}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\vspace{6pt}
\appendixtitles{yes}
\appendix
\section{Mortgage Data Details}\label{a_sec:mort}
The US HMDA law, originally enacted in 1975, requires many financial institutions that originate mortgage products to provide certain loan-level data about many types of mortgage-related products on an annual basis. This information is first provided to the CFPB, which subsequently releases some of the data to the public. Regulators often use HMDA data to, ``... show whether lenders are serving the housing needs of their communities; they give public officials information that helps them make decisions and policies; and they shed light on lending patterns that could be discriminatory'' (see \href{https://www.consumerfinance.gov/data-research/hmda/}{Mortgage data (HMDA)}). In addition to regulatory use, public advocacy groups use these data for similar purposes, and the lenders themselves use the data to benchmark their community outreach relative to their peers. The publicly available data that the CFPB releases includes information such as the lender, the type of loan, loan amount, LTV ratio, DTI ratio, and other important financial descriptors. The data also include information on each borrower and co-borrower’s race, ethnicity, gender, and age. Because the data includes information on these protected class characteristics, certain measures that can be indicative of discrimination in lending can be calculated directly using the HDMA data. Ultimately, the HMDA data represent the most comprehensive source of data on highly-regulated mortgage lending that is publicly available, which makes it an ideal dataset to use for the types of analyses set forth in Sections~\ref{sec:m_and_m}~and~\ref{sec:res}.
\section{Selected Algorithmic Details}\label{a_sec:algo}
\vspace{-6pt}
\subsection{Notation}\label{a_ssec:not}
To facilitate descriptions of data, modeling, and other post-hoc techniques, notation for input and output spaces, datasets, and models is defined.
\subsubsection{Spaces}
\begin{itemize}[leftmargin=*,labelsep=5.8mm]
\item Input features come from the set $\mathcal{X}$ contained in a \textit{P}-dimensional input space, $\mathcal{X} \subset \mathbb{R}^P$. An~arbitrary, potentially unobserved, or future instance of $\mathcal{X}$ is denoted $\mathbf{x}$, $\mathbf{x} \in \mathcal{X}$.
\item Labels corresponding to instances of $\mathcal{X}$ come from the set $\mathcal{Y}$.
\item Learned output responses of models are contained in the set $\mathcal{\hat{Y}}$. % For regression models, the set $\mathcal{\hat{Y}}_r$ is also contained in a $C$-dimensional output space, $\mathcal{\hat{Y}}_r \subset \mathbb{R}^{C_r}$. For classification models, the set $\mathcal{\hat{Y}}_c$ typically contains a column vector for each unique class in $\mathcal{Y}$. Hence, $\mathcal{\hat{Y}}_c$ is contained in a $C'$-dimensional output space, $\mathcal{\hat{Y}}_c \subset \mathbb{R}^{C'_c}$.
\end{itemize}
\subsubsection{Data}
\begin{itemize}[leftmargin=*,labelsep=5.8mm]
\item An input dataset $\mathbf{X}$ is composed of observed instances of the set $\mathcal{X}$ with a corresponding dataset of labels $\mathbf{Y}$, observed instances of the set $\mathcal{Y}$.
\item Each $i$-th observed instance of $\mathbf{X}$ is denoted as $\mathbf{x}^{(i)} = $
$[x_0^{(i)}, x_1^{(i)}, \dots, x_{\textit{P}-1}^{(i)}]$, with corresponding $i$-th labels in $\mathbf{Y}, \mathbf{y}^{(i)}$, and corresponding predictions in $\mathbf{\hat{Y}}, \mathbf{\hat{y}}^{(i)}$. % = [y_0^{(i)}, y_1^{(i)}, \dots, y_{\textit{C}-1}^{(i)}]$.
\item $\mathbf{X}$ and $\mathbf{Y}$ consist of $N$ tuples of observed instances: $[(\mathbf{x}^{(0)},\mathbf{y}^{(0)}), (\mathbf{x}^{(1)},\mathbf{y}^{(1)}), \dots,(\mathbf{x}^{(N-1)},\mathbf{y}^{(N-1)})]$. %\\ $\mathbf{x}^{(i)} \in \mathcal{X}$, $\mathbf{y}^{(i)} \in \mathcal{Y}$.
\item Each $j$-th input column vector of $\mathbf{X}$ is denoted as $X_j = [x_{j}^{(0)}, x_{j}^{(1)}, \dots, x_{j}^{(N-1)}]^T$.
\end{itemize}
\subsubsection{Models}
\begin{itemize}[leftmargin=*,labelsep=5.8mm]
\item A type of ML model $g$, selected from a hypothesis set $\mathcal{H}$, is trained to represent an unknown signal-generating function $f$ observed as $\mathbf{X}$ with labels $\mathbf{Y}$ using a training algorithm $\mathcal{A}$:
$ \mathbf{X}, \mathbf{Y} \xrightarrow{\mathcal{A}} g$, such that $g \approx f$.
\item $g$ generates learned output responses on the input dataset $g(\mathbf{X}) = \mathbf{\hat{Y}}$, and on the general input space $g(\mathcal{X}) = \mathcal{\hat{Y}}$.
\item A model to be explained or tested for discrimination is denoted as $g$.
\end{itemize}
\subsection{Monotonic Gradient Boosting Machine Details}\label{a_ssec:mgbm}
For some $g^\text{MGBM}$ model (see Equation (\ref{eq:gbm})), monotonic splitting rules, $\Theta^{\text{mono}}_b$, are selected in a greedy, additive fashion by minimizing a regularized loss function, $\mathcal{L}$, that considers known target labels, $\mathbf{y}$, the predictions of all subsequently trained trees in $g^{\text{MGBM}}$, $g^{\text{MGBM}}_{b-1}(\mathbf{X})$, and the $b$-th tree splits applied to some instance $\mathbf{x}$, $T_b(\mathbf{x};\Theta^\text{mono}_b)$, in a numeric error function (e.g., squared error, Huber error), $l$, in addition to a regularization term that penalizes complexity in the $b$-th tree, $\Omega(T_b)$. For the $b$-th iteration over $N$ instances, $\mathcal{L}_{b}$, can generally be defined as:
\begin{equation}
\begin{aligned}
\mathcal{L}_{b} =\sum_{i=0}^{N-1}l(y^{(i)}, g^{\text{MGBM}}_{b-1}(\mathbf{x}^{(i)}), T_b(\mathbf{x}^{(i)};\Theta^\text{mono}_b)) + \Omega(T_b)\\
\end{aligned}
\end{equation}
In addition to $\mathcal{L}$, $g^\text{MGBM}$ training is characterized by monotonic splitting rules and constraints on tree node weights. Each binary splitting rule in $T_b$, $\theta_{b,j,k} \in \Theta_b$, is associated with a feature, $X_j$, is the $k$-th split associated with $X_j$ in $T_b$, and results in left, $L$, and right, $R$, child nodes with a numeric weights, $\{w_{b,j,k,L}, w_{b,j,k,R}\}$. For terminal nodes, $\{w_{b,j,k,L}, w_{b,j,k,R}\}$ can be direct numeric components of some $g^\text{MGBM}$ prediction. For two values, $x^{\alpha}_j$ and $x^{\beta}_j$, of some feature $X_j$, where $x^{\alpha}_j \le x^{\beta}_j$, $g^{\text{MGBM}}$ is positive monotonic with respect to $X_j$ if $g^{\text{MGBM}}(x^{\alpha}_j) \le g^{\text{MGBM}}(x^{\beta}_j) ~\forall ~\{x^{\alpha}_j \le x^{\beta}_j\} \in X_j$. The following rules and constraints ensure positive monotonicity in $\Theta_b^\text{mono}$:
\begin{enumerate}[leftmargin=*,labelsep=4.9mm]
\item For the first and highest split in $T_b$ involving $X_j$, any $\theta_{b,j,0}$ resulting in $T(x_j; \theta_{b,j,0}) = \{w_{b,j,0,L}, w_{b,j,0,R}\}$ where $w_{b,j,0,L} > w_{b,j,0,R}$, is not considered.
\item For any subsequent left child node involving $X_j$, any $\theta_{b,j, k\ge1}$ resulting in $T(x_j; \theta_{b,j,k\ge1}) = \{w_{b,j,k\ge1,L}, w_{b,j,k\ge1,R}\}$ where $w_{b,j,k\ge1,L} > w_{b,j,k\ge1,R}$, is not considered.
\item Moreover, for any subsequent left child node involving $X_j$, $T(x_j; \theta_{b,j,k\ge1}) = \{w_{b,j,k\ge1,L}, w_{b,j,k\ge1,R}\}$, $\{w_{b,j,k\ge1,L}, w_{b,j,k\ge1,R}\}$ are bound by the associated $\theta_{b,j,k-1}$ set of node weights, $\{w_{b,j,k-1,L}, w_{b,j,k-1, R}\}$, such that $ \{w_{b,j,k\ge1,L}, w_{b,j,k\ge1,R}\} < \frac{w_{b,j,k-1,L} + w_{b,j,k-1,R}}{2}$.
\item (1) and (2) are also applied to all right child nodes, except that for right child nodes $ w_{b,j,k,L} \le w_{b,j,k,R}$ and $\{w_{b,j,k\ge1,L}, w_{b,j,k\ge1,R}\} \ge \frac{w_{b,j,k-1,L} + w_{b,j,k-1,R}}{2}$.
\end{enumerate}
Note that for any one $X_j$ and subtree in $g^{\text{MGBM}}$, left subtrees will always produce lower predictions than right subtrees, and that any $g^{\text{MGBM}}(\mathbf{x})$ is an addition of each full $T_b$ prediction, with the application of a monotonic logit or softmax link function for classification problems. Moreover, each tree's root node corresponds to some constant node weight that by definition obeys monotonicity constraints, $ T(x^{\alpha}_j; \theta_{b,0}) = T(x^{\beta}_j; \theta_{b,0}) = w_{b,0}$. Together these additional splitting rules and node weight constraints ensure that $g^{\text{MGBM}}(x^{\alpha}_j) \le g^{\text{MGBM}}(x^{\beta}_j) ~\forall~ \{x^{\alpha}_j \le x^{\beta}_j\} \in X_j$. For a negative monotonic constraint, i.e., $g^{\text{MGBM}}(x^{\alpha}_j) \ge g^{\text{MGBM}}(x^{\beta}_j) ~\forall~ \{x^{\alpha}_j \le x^{\beta}_j\} \in X_j$, left and right splitting rules and node weight constraints are switched. Also consider that MGBM models with independent monotonicity constraints between some $X_j$ and $\mathbf{y}$ likely restrict non-monotonic interactions between multiple $X_j$. Moreover, if monotonicity constraints are not applied to all $X_j \in \mathbf{X}$, any strong non-monotonic signal in training data associated with some important $X_j$ maybe forced onto some other arbitrary unconstrained $X_j$ under some $g^{MGBM}$ models, compromising the end goal of interpretability.
\subsection{Explainable Neural Network Details}\label{a_ssec:xnn}
$g^{\text{XNN}}$ is comprised of 3 meta-layers:
\begin{enumerate}[leftmargin=*,labelsep=4.9mm]
\item The first and deepest meta-layer, composed of $K$ linear $\sum_j\beta_{k,j}x_j$ hidden units (see Equation (\ref{eq:xnn})), which should learn higher magnitude weights for each important input, $X_j$, is known as the \textit{projection layer}. It is fully connected to each input $X_j$. Each hidden unit in the projection layer may optionally include a bias term.
\item The second meta-layer contains $K$ hidden and separate $n_k$ ridge functions, or \textit{subnetworks}. Each $n_k$ is a neural network itself, which can be parametrized to suit a given modeling task. To facilitate direct interpretation and visualization, the input to each subnetwork is the 1-dimensional output of its associated projection layer $\sum_j\beta_{k,j}x_j$ hidden unit. Each $n_k$ can contain several bias terms.
\item The output meta-layer, called the \textit{combination layer}, is an output neuron comprised of a global bias term, $\mu_0$, and the $K$ weighted 1-dimensional outputs of each subnetwork, $\gamma_kn_k(\sum_j\beta_{k,j}x_j)$. Again, each $n_k$ subnetwork output into the combination layer is restricted to 1-dimension for interpretation and visualization purposes.
\end{enumerate}
\begin{figure}[H]
\centering
\includegraphics[width=12cm]{img/xnn_arch.png}
\caption{{A general diagram} of an XNN with three metalayers: the bottom combination layer with $K$ linear $\sum_j\beta_{k,j}x_j$ hidden units, the middle metalayer with with $K$ hidden and separate $n_k$ ridge functions, and the output combination layer that generates $g^{\text{XNN}}(\mathbf{X})$ predictions. Figure adapted from \citet{wf_xnn}.}
\label{fig:xnn_arch}%Figure A1 is not mentioned in the maunscript
\end{figure}
\subsection{One-dimensional Partial Dependence and Individual Conditional Expectation Details}\label{a_ssec:pd_ice}
Following \citet{esl} a single input feature, $X_j \in \mathbf{X}$, and its complement set, $\mathbf{X}_{\mathcal{P} \setminus \{j\}} \in \mathbf{X}$, where $X_j \cup \mathbf{X}_{\mathcal{P} \setminus \{j\}} = \mathbf{X}$ is considered. $\text{PD}(X_j, g)$ for a given $X_j$ is the estimated average output of the learned function $g(\mathbf{X})$ when all the observed instances of $X_j$ are set to a constant $x^\gamma \in \mathcal{X}$ and $\mathbf{X}_{\mathcal{P} \setminus \{j\}}$ is left unchanged. $\text{ICE}(x_j, g)$ for a given instance $\mathbf{x}$ and feature $x_j$ is estimated as the output of $g(\mathbf{x})$ when $x_j$ is set to a constant $x^\gamma \in \mathcal{X}$ and all other features $\mathbf{x} \in \mathbf{X}_{\mathcal{P} \setminus \{j\}}$ are left untouched. PD and ICE curves are usually plotted over some set of constants drawn from $\mathcal{X}$, as displayed in Section~\ref{ssec:int_ex_mort} and Appendix \ref{a_ssec:int_ex_sim}. Due to known problems for PD in the presence of strong correlation and interactions, PD should not be used alone. PD should be paired with ICE or be replaced with accumulated local effect (ALE) plots \cite{ice_plots,ale_plot}.
\subsection{Shapley Value Details}\label{a_ssec:shap}
For some instance $\mathbf{x} \in \mathcal{X}$, Shapley explanations take the form:
\begin{equation}
\label{eq:shap_additive}
\begin{aligned}
g(\mathbf{x}) = \phi_0 + \sum_{j=0}^{j=\mathcal{P} - 1} \phi_j z_j
\end{aligned}
\end{equation}
In Equation (\ref{eq:shap_additive}), $\mathbf{z} \in \{0,1\}^\mathcal{P}$ is a binary representation of $\mathbf{x}$ where 0 indicates missingness. Each Shapley value, $\phi_j$, is the local feature contribution value associated with $x_j$, and $\phi_0$ is the average of $g(\mathbf{X})$. Each $\phi_j$ is a weighted combination of model predictions with $x_j$, $g_x(S \cup \{j\})$, and the model predictions without $x_j$, $g_x(S)$, for every possible subset of features $S$ not including $j$, $S \subseteq \mathcal{P} \setminus \{j\}$, where $g_x$ incorporates the mapping between $\mathbf{x}$ and the binary vector $\mathbf{z}$.
\begin{equation}
\label{eq:shap_contrib}
\begin{aligned}
\phi_{j} = \sum_{S \subseteq \mathcal{P} \setminus \{j\}}\frac{|S|!(\mathcal{P} -|S| -1)!}{\mathcal{P}!}[g_x(S \cup \{j\}) - g_x(S)]
\end{aligned}
\end{equation}
Local, per-instance explanations using Shapley values tend to involve ranking $x_j \in \mathbf{x}$ by $\phi_j$ values or delineating a set of the $X_j$ names associated with the $k$-largest $\phi_j$ values for some $\mathbf{x}$, where $k$ is some small positive integer, say five. Global explanations are typically the absolute mean of the $\phi_j$ associated with a given $X_j$ across all of the instances in some set $\mathbf{X}$.
\section{Types of Machine Learning Discrimination in US Legal and Regulatory Settings}\label{a_sec:types}
It is important to explain and draw a distinction between the two major types of discrimination recognized in US legal and regulatory settings, disparate treatment (DT) and disparate impact (DI). DT occurs most often in an algorithmic setting when a model explicitly uses protected class status (e.g., race, sex) as an input feature or uses a feature that is so similar to protected class status that it essentially proxies for class membership. With some limited exceptions, the use of these factors in an algorithm is illegal under several statutes in the US. DI occurs when some element of a decisioning process includes a {facially neutral} factor (i.e, a reasonable and valid predictor of response) that results in a disproportionate share of a protected class receiving an unfavorable outcome. In modeling, this is most typically driven by a statistically important feature that is distributed unevenly across classes, which causes more frequent unfavorable outcomes for the protected class. However, other factors, such as hyperparameter or algorithm choices, can drive DI. Crucially, legality hinges on whether changing the model, for example exchanging one feature for another or altering the hyperparameters of an algorithm, can lead to a similarly predictive model with lower DI.
\section{Practical vs. Statistical Significance for Discrimination Testing}\label{a_sec:prac_v_stat}
A finding of \textit{practical significance} means that discovered disparity is not only statistically significant, but also passes beyond a chosen threshold that would constitute {prima facie} evidence of illegal discrimination. Practical significance acknowledges that any large dataset is likely to show statistically significant differences in outcomes by class, even if those differences are not truly meaningful. It further recognizes that there are likely to be situations where differences in outcomes are beyond a model user’s ability to correct them without significantly degrading the quality of the model. Moreover, practical significance is also needed by model builders and compliance personnel to determine whether a model should undergo remediation efforts before it is put into production. Unfortunately, guidelines for practical significance, i.e., the threshold at which any statistically significant disparity would be considered evidence of illegal discrimination, are not as frequently codified as the standards for statistical significance. One exception, however, is in employment discrimination analyses, where~the US Equal Employment Opportunity Commission (EEOC) has stated that if the AIR is below 0.80 and statistically significant, then this constitutes {prima facie} evidence of discrimination, which the model user must rebut in order for the DI not to be considered illegal discrimination. (Importantly, the standard of 0.80 is not a law, but a rule of thumb for agencies tasked with enforcement of discrimination laws. ``Adoption of Questions and Answers To Clarify and Provide a Common Interpretation of the Uniform Guidelines on Employee Selection Procedures,'' Federal Register, Volume 44, Number 43 (1979).) It is important to note that the 0.80 measure of practical significance, also known as the \textit{80\% rule} and the \textit{4/5ths rule}, is explicitly used in relation to AIR, and it is not clear that the use of this threshold is directly relevant to testing fairness for measures other than the AIR.
The legal thresholds for determining statistical significance is clearer and more consistent than that for practical significance. The first guidance in US courts occurred in a case involving discrimination in jury selection, \textit{Castaneda vs. Partida} (430 US 482 - Supreme Court (1977)). Here, the US Supreme Court wrote that, ``As a general rule for such large samples, if the difference between the expected value and the observed number is greater than two or three standard deviations, then the hypothesis that the jury drawing was random would be suspect to a social scientist.'' This ``two or three standard deviations'' test was then applied to employment discrimination in \textit{Hazelwood School Districts vs. United States} (433 US 299 (1977)). Out of this, a 5\% two-sided test (z = 1.96), or an equivalent 2.5\% one-sided test, has become a common standard for determining whether evidence of disparities is statistically significant.
\section{Additional Simulated Data Results}\label{a_sec:res_sim}
As seen in Section~\ref{ssec:c_v_uc_sim}, little or no trade-off is required in terms of model to fit to use the constrained models. Hence, intrinsic interpretability, post-hoc explainability, and discrimination are explored further for the $g^\text{MGBM}$ and $g^\text{XNN}$ models in Appendices \ref{a_ssec:int_ex_sim} and \ref{a_ssec:dis_sim}. Intrinsic interpretability for $g^\text{MGBM}$ is evaluated with PD and ICE, and post-hoc explainability is highlighted via global and local Shapley explanations. For $g^\text{XNN}$, Shapley explanation techniques are also used to generate global and local feature importance to augment interpretability results exhibited in Section \ref{ssec:xnn_sim}. Both~$g^\text{MGBM}(\mathbf{X})$ and $g^\text{XNN}(\mathbf{X})$ are evaluated for discrimination using AIR, ME, SMD, and other measures.
\subsection{Interpretability and Post-hoc Explanation Results}\label{a_ssec:int_ex_sim}
Global mean absolute Shapley value feature importance for $g^{\text{MGBM}}(\mathbf{X})$ on the simulated test data is displayed in Figure \ref{fig:sim_mgbm_glob}.
\begin{figure}[H]
\centering
\captionsetup{width=16cm}
\centering
\includegraphics[width=12cm]{img/sim_mgbm_glob.png}
\caption{Global mean absolute Tree SHAP feature importance for $g^\text{MGBM}(\mathbf{X})$ on the simulated test data.}
\label{fig:sim_mgbm_glob}
\end{figure}
As expected, the $X_{\text{Friedman},j}$ features from the original \citet{friedman1979tree} and \citet{friedman1991multivariate} formula are the main drivers of $g^\text{MGBM}(\mathbf{X})$ predictions, with encoded versions of the augmented categorical and binary features contributing less on average to $g^\text{MGBM}(\mathbf{X})$ predictions.
Figure \ref{fig:sim_mgbm_glob_pdp_ice} highlights PD, ICE, and histograms of the most important features from Figure \ref{fig:sim_mgbm_glob}.
\begin{figure}[H]
\centering
\includegraphics[width=15cm]{img/sim_mgbm_glob_pdp_ice.png}
\caption{PD, ICE for ten instances across selected percentiles of $g^\text{MGBM}(\mathbf{X})$, and histograms for the three most important input features of $g^\text{MGBM}$ on the simulated test data.}
\label{fig:sim_mgbm_glob_pdp_ice}
\end{figure}
$X_{\text{Friedman},1}$, $X_{\text{Friedman},2}$, and $X_{\text{Friedman},4}$ were positively monotonically constrained under $g^\text{MGBM}$ for the simulated data, and positive monotonicity looks to be confirmed on average with PD and at numerous local percentiles of $g^\text{MGBM}(\mathbf{X})$ with ICE. As the PD curves generally follow the patterns of the ICE curves, PD is likely an accurate representation of average feature behavior for $X_{\text{Friedman},1}$, $X_{\text{Friedman},2}$, and $X_{\text{Friedman},4}$. Also because PD and ICE curves do not obviously diverge, $g^\text{MGBM}$ is likely not modeling strong interactions, despite the fact that known interactions are included in the simulated data signal-generating function in Equation (\ref{eq:sim}). The one-dimensional monotonic constraints may hinder the modeling of non-monotonic interactions, but do not strongly affect overall $g^\text{MGBM}$ accuracy, perhaps due to the main drivers, $X_{\text{Friedman},1}$, $X_{\text{Friedman},2}$, and $X_{\text{Friedman},4}$, all being constrained in the same direction and able to weakly interact as needed.
Local Shapley values for records at the 10{th}, 50{th}, and 90{th} percentiles of $g^\text{MGBM}(\mathbf{X})$ in the simulated test data are displayed in Figure \ref{fig:sim_mgbm_loc}.
\begin{figure}[H]
\centering
\includegraphics[width=15cm]{img/sim_mgbm_loc.png}
\caption{Tree SHAP values for three instances across selected percentiles of $g^\text{MGBM}(\mathbf{X})$ for the simulated test data.}
\label{fig:sim_mgbm_loc}
\end{figure}
The Shapley values in Figure \ref{fig:sim_mgbm_loc} appear to be a logical result. For the lower prediction at the 10{th} percentile of $g^\text{MGBM}(\mathbf{X})$, the largest local contributions are negative and the majority of local contributions are also negative. At the median of $g^\text{MGBM}(\mathbf{X})$, local contributions are roughly split between positive and negative values, and at the 90{th} of $g^\text{MGBM}(\mathbf{X})$, most large contributions are positive. In each case, large local contributions generally follow global importance results in Figure \ref{fig:sim_mgbm_glob} as well.
Figure \ref{fig:sim_xnn_glob} shows global mean absolute Shapley feature importance for $g^\text{XNN}(\mathbf{X})$ on the simulated test data, using the approximate Deep SHAP technique.
\begin{figure}[H]
\centering
\captionsetup{width=16cm}
\centering
\includegraphics[width=12cm]{img/sim_xnn_glob.png}
\caption{Global mean absolute Deep SHAP feature importance for $g^\text{XNN}(\mathbf{X})$ on the simulated test data.}
\label{fig:sim_xnn_glob}
\end{figure}
Like $g^\text{MGBM}$, $g^\text{XNN}$ ranks the $X_{\text{Friedman},j}$ features higher in terms of importance than the categorical and binary features. The consistency between the feature rankings of $g^\text{MGBM}$ and $g^\text{XNN}$ is somewhat striking, given their different hypothesis families and architectures. Both $g^\text{MGBM}$ and $g^\text{XNN}$ rank $X_{\text{Friedman},1}$, $X_{\text{Friedman},2}$, and $X_{\text{Friedman},4}$ as the most important features, both place $X_{\text{Categorical},2}$ and $X_{\text{Categorical},3}$ above the $X_{\text{Binary},1}$ and $X_{\text{Binary},2}$ features, both rank $X_{\text{Binary},1}$ above $X_{\text{Binary},2}$, and both place the least importance on $X_{\text{Categorical},4}$ and $X_{\text{Categorical},0}$.
Local Deep SHAP feature importance in Figure \ref{fig:sim_xnn_loc} supplements the global interpretability of $g^\text{XNN}$ displayed in Figures \ref{fig:sim_xnn_glob} and \ref{fig:sim_xnn_glob_ridge}. Local Deep SHAP values are extracted from the projection layer of $g^\text{XNN}$ and reported in the probability space. Deep SHAP values can be calculated for any arbitrary $g^\text{XNN}(\mathbf{x})$, allowing for detailed, local summarization of individual model predictions.
\begin{figure}[H]
\centering
\includegraphics[width=15cm]{img/sim_xnn_loc.png}
\caption{Deep SHAP values for three instances across selected percentiles of $g^\text{XNN}(\mathbf{X})$ on the simulated test data.}
\label{fig:sim_xnn_loc}
\end{figure}
As expected, Deep SHAP values generally increase from the 10{th} percentile of $g^\text{XNN}(\mathbf{X})$ to the 90{th} percentile of $g^\text{XNN}(\mathbf{X})$, with primarily important global drivers of model behavior contributing to the selected local $g^\text{XNN}(\mathbf{x})$ predictions.
\subsection{Discrimination Testing Results}\label{a_ssec:dis_sim}
Tables \ref{tab:sim_disc}{a,b} show the results of the disparity tests using the simulated data for two hypothetical sets of class-control groups. Several measures of disparities are shown, with the SMDs calculated using the probabilities from $g^{\text{MGBM}}(\mathbf{X})$ and $g^{\text{XNN}}(\mathbf{X})$, FNRs, their ratios, MEs, and AIRs calculated using a binary outcome based on a cutoff of 0.6 (anyone with probabilities of 0.6 or greater receives the favorable outcome).
Since $g^{\text{MGBM}}$ and $g^{\text{XNN}}$ assume that a higher score is favorable (as might be the case if the model were predicting responses to marketing offers), one might consider the relative FNRs as a measure of the class-control disparities. Table \ref{tab:sim_disc}{b} shows that protected group 1 has higher relative FNRs under $g^{\text{XNN}}$ (1.13 vs. 1.06). However, in Table \ref{tab:sim_disc}{a} the overall FNRs were lower for $g^{\text{XNN}}$ (0.357 vs. 0.401). This illustrates a danger in considering relative class-control measures in isolation when comparing across models: despite the $g^\text{MGBM}$ appearing to be a relatively fairer model, more protected group 1 members experience negative outcomes using $g^{\text{MGBM}}$. This is because FNR accuracy improves for both the protected group 1 and control group 1, but members of control group 1 benefit more than those in protected group 1. Of course, the choice of which model is truly fairer is a policy question.
%\begin{table}[H]
% \hskip-10pt
% \begin{subtable}{0.45\textwidth}
% \captionsetup{width=7cm}
% \caption{Group size, accuracy, and FNR for $g^\text{MGBM}(\mathbf{X})$ and $g^\text{XNN}(\mathbf{X})$ on the simulated test data.}
% \centering
% \resizebox{\columnwidth}{!}{%
% \begin{tabular}{ccccc}
% \toprule
% \textbf{Class} & $\mathbf{N}$ & \textbf{Model} & \textbf{Accuracy}$\uparrow$ & \textbf{FNR}$\downarrow$ \\
% \midrule
% Protected 1 & 3,057 & \begin{tabular}{@{}c@{}} $g^\text{MGBM}$ \\ $g^\text{XNN}$\end{tabular} & \begin{tabular}{@{}c@{}} 0.770 \\ 0.771\end{tabular} & \begin{tabular}{@{}c@{}} 0.401 \\ 0.357\end{tabular} \\
% \midrule
% Control 1 & 16,943 & \begin{tabular}{@{}c@{}} $g^\text{MGBM}$ \\ $g^\text{XNN}$\end{tabular} & \begin{tabular}{@{}c@{}} 0.739 \\ 0.756\end{tabular} & \begin{tabular}{@{}c@{}} 0.378 \\ 0.314\end{tabular} \\
% \midrule
% Protected 2 & 9,916 & \begin{tabular}{@{}c@{}} $g^\text{MGBM}$ \\ $g^\text{XNN}$\end{tabular} & \begin{tabular}{@{}c@{}} 0.758 \\ 0.762\end{tabular} & \begin{tabular}{@{}c@{}} 0.331 \\ 0.302\end{tabular} \\
% \midrule
% Control 2 & 10,084 & \begin{tabular}{@{}c@{}} $g^\text{MGBM}$ \\ $g^\text{XNN}$\end{tabular} & \begin{tabular}{@{}c@{}} 0.729 \\ 0.756\end{tabular} & \begin{tabular}{@{}c@{}} 0.420 \\ 0.332\end{tabular} \\
% \bottomrule
% \label{tab:sim_n_acc}
% \end{tabular}%
% }
% \end{subtable}
% \hspace{5pt}
% \begin{subtable}{0.45\textwidth}
% \vspace{-11pt}
% \captionsetup{width=10cm}
% \caption{%
% \leftskip=45pt
% AIR, ME, SMD, and FNR ratio for $g^\text{MGBM}(\mathbf{X})$ and $g^\text{XNN}(\mathbf{X})$ on the simulated test data.}
% \centering
% \scalebox{0.95}{
% \begin{tabular}{ccccccc}
% \toprule
% \textbf{Model} & \begin{tabular}[x]{@{}c@{}}\textbf{Protected}\\\textbf{Class}\end{tabular} & \begin{tabular}[x]{@{}c@{}}\textbf{Control}\\\textbf{Class}\end{tabular} & \textbf{AIR}$\uparrow$ & \textbf{ME}$\downarrow$ & \textbf{SMD}$\downarrow$ & \begin{tabular}[x]{@{}c@{}}\textbf{FNR}\\\textbf{Ratio}\end{tabular}$\downarrow$\\
% \midrule
% $g^\text{MGBM}$ & \begin{tabular}{@{}c@{}} 1 \\ 2\end{tabular} & \begin{tabular}{@{}c@{}} 1 \\ 2\end{tabular} & \begin{tabular}{@{}c@{}} 0.752 \\ 1.10\end{tabular} & \begin{tabular}{@{}c@{}} 9.7\% \\ -3.6\%\end{tabular} & \begin{tabular}{@{}c@{}} -0.206 \\ 0.106\end{tabular} & \begin{tabular}{@{}c@{}} 1.06 \\ 0.788\end{tabular}\\
% \midrule
% $g^\text{XNN}$ & \begin{tabular}{@{}c@{}} 1 \\ 2\end{tabular} & \begin{tabular}{@{}c@{}} 1 \\ 2\end{tabular} & \begin{tabular}{@{}c@{}} 0.727 \\ 0.976\end{tabular} & \begin{tabular}{@{}c@{}} 12.0\%\\ 1.0\%\end{tabular} & \begin{tabular}{@{}c@{}} -0.274 \\ 0.001\end{tabular} & \begin{tabular}{@{}c@{}} 1.13 \\ 0.907\end{tabular} \\
% \bottomrule
% \label{tab:sim_disc_detail}
% \end{tabular}
% }
% \end{subtable}
% \captionsetup{width=10cm}
% \caption{Discrimination measures for the simulated test data. Arrows indicate the direction of improvement for each measure.}
% \label{tab:sim_disc}
%\end{table}
\begin{table}[H]
\centering
\caption{Discrimination measures for the simulated test data. Arrows indicate the direction of improvement for each measure.}
\label{tab:sim_disc}
\begin{tabular}{ccccc}