-
Notifications
You must be signed in to change notification settings - Fork 11
/
mind.py
97 lines (70 loc) · 2.47 KB
/
mind.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
TensorFlow Implementation of MIND ([1]) under Spearman rank correlation constraints.
[1] Kom Samo, Y. (2021). Inductive Mutual Information Estimation: A Convex Maximum-Entropy Copula Approach . <i>Proceedings of The 24th International Conference on Artificial Intelligence and Statistics</i>, in <i>Proceedings of Machine Learning Research</i> 130:2242-2250 Available from https://proceedings.mlr.press/v130/kom-samo21a.html.
"""
import numpy as np
from kxy.misc.tf import CopulaLearner
def copula_entropy(z, subsets=[]):
'''
Estimate the entropy of the copula distribution of a d-dimensional random vector using MIND ([1]) with Spearman rank correlation constraints.
Parameters
----------
z : np.array
Vector whose rows are samples from the d-dimensional random vector and columns its coordinates.
Returns
-------
ent : float
The estimated copula entropy.
'''
if len(z.shape)==1 or z.shape[1]==1:
return 0.0
d = z.shape[1]
cl = CopulaLearner(d, subsets=subsets)
cl.fit(z)
ent = min(cl.copula_entropy, 0.0)
return ent
def mutual_information(y, x):
'''
Estimate the mutual information between two random vectors using MIND ([1]) with Spearman rank correlation constraints.
Parameters
----------
y : np.array
Vector whose rows are samples from the d-dimensional random vector and columns its coordinates.
x : np.array
Vector whose rows are samples from the d-dimensional random vector and columns its coordinates.
Returns
-------
mi : float
The estimated mutual information.
'''
y = y[:, None] if len(y.shape)==1 else y
x = x[:, None] if len(x.shape)==1 else x
z = np.concatenate([y, x], axis=1)
huy = copula_entropy(y)
hux = copula_entropy(x)
huz = copula_entropy(z)
mi = max(huy+hux-huz, 0.0)
return mi
def run_d_dimensional_gaussian_experiment(d, rho, n=1000):
'''
'''
# Cholesky decomposition of corr = np.array([[1., rho], [rho, 1.]])
L = np.array([[1., 0.], [rho, np.sqrt(1.-rho*rho)]])
y = np.empty((n, d))
x = np.empty((n, d))
for i in range(d):
u = np.random.randn(n, 2)
z = np.dot(L, u.T).T
y[:, i] = z[:, 0].copy()
x[:, i] = z[:, 1].copy()
estimated_mi = mutual_information(y, x)
true_mi = -d*0.5*np.log(1.-rho*rho)
return estimated_mi, true_mi
if __name__ == '__main__':
rho = 0.95
d = 20
estimated_mi, true_mi = run_d_dimensional_gaussian_experiment(d, rho)
print('%dd Gaussian Mutual Information: Estimated %.4f, True (theoretical) %.4f' % (\
d, estimated_mi, true_mi))