fix bug in dataset indexing

Author: Dimitri Coukos

.DS_Store

@@ -11,3 +11,5 @@ Experiment 1:
 - Depth: {2, 4, 6, 8, 10, 12... while perf increasing}
 - Input Data: {Masif identifiers, Electrostatics, + Shape Index, +Rotated Positional Data}
 - Uses SeLU because ReLU kills the learning...
+
+Observations & Results:

README.md

@@ -12,6 +12,8 @@
 File to generate the dataset from the ply files.
 
 '''
+
+
 def convert_data(path_to_raw='./structures/', n=None, prefix='full'):
     '''Generate raw unprocessed torch file to generate pyg datasets with fewer
         candidates.
@@ -160,7 +162,7 @@ def read_ply(path, learn_iface=True):
 
     x = ([torch.tensor(data['vertex'][axis]) for axis in ['charge', 'hbond', 'hphob']])
     x = torch.stack(x, dim=-1)
-    y = None  #what the fuck
+    y = None
 
     y = [torch.tensor(data['vertex']['iface'])]
     y = torch.stack(y, dim=-1)
@@ -254,16 +256,15 @@ def __init__(self, root='./datasets/{}/'.format(p.dataset), pre_transform=None,
         super(StructuresDataset, self).__init__(root, transform, pre_transform)
         self.has_nan = []
 
-
     @property
     def raw_file_names(self):
-        n_files = len(glob('{}/raw/full_structure_*'.format(self.root, p.dataset)))
+        n_files = len(glob('{}/raw/full_structure_*'.format(self.root)))
         return ['full_structure_{}.pt'.format(idx) for idx in range(0, n_files)]
 
     @property
     def processed_file_names(self):
-        n_files = len(glob('./datasets/{}/processed/data*'.format(p.dataset)))
-        return ['data_0.pt']  # right order
+        n_files = len(glob('{}/processed/data*'.format(self.root)))
+        return ['data_{}.pt'.format(i) for i in range(0, n_files)]  # right order
 
     def download(self):
         pass
@@ -285,8 +286,8 @@ def process(self):
             torch.save(data, osp.join(self.processed_dir, 'data_{}.pt'.format(i)))
             i += 1
 
-    def __len__(self):
-        return len(self.processed_file_names)
+    def len(self):
+        return len(self.processed_paths)
 
     def get(self, idx):
         data = torch.load(osp.join(self.processed_dir, 'data_{}.pt'.format(idx)))

dataset.py

@@ -84,6 +84,7 @@
 # ---- Training ----
 
 for model_n, model in enumerate(models):
+
     model.to(device)
     optimizer = torch.optim.Adam(model.parameters(), lr=learn_rate, weight_decay=p.weight_decay)
 # ------------ TRAINING NEW BLOCK --------------------------
@@ -94,11 +95,12 @@
         masked_loader = DataLoader(maskedset, shuffle=False, batch_size=p.test_batch_size)
 
         data = next(iter(train_loader))
-        ns = NeighborSampler(next(iter(train_loader)), 0.92, 9, batch_size=1000)
+        ns = NeighborSampler(next(iter(train_loader)), 0.4, 9, batch_size=1)
 
         # error with NeighborSampler:
         # neighbor sampler does not seem to be iterable like in the example.
-
+        for dataflow in ns():
+            print(dataflow)
         model.train()
         first_batch_labels = torch.Tensor()
         pred = torch.Tensor()

model_22.py

@@ -478,5 +478,35 @@ def test(mask):
     print('Epoch: {:02d}, Loss: {:.4f}, Test: {:.4f}'.format(
         epoch, loss, test_acc))
 
-
+# ---------------------- Trying to use datastructures ----------------------------
+import torch
 from dataset import StructuresDataset
+from transforms import *
+from torch_geometric.transforms import *
+from models import TwoConv
+import params as p
+
+device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
+cpu = torch.device('cpu')
+# reproducibility
+torch.manual_seed(p.random_seed)
+np.random.seed(p.random_seed)
+learn_rate = p.learn_rate
+
+
+model = TwoConv(3, heads=p.heads).to(device)
+optimizer = torch.optim.Adam(model.parameters(), lr=learn_rate, weight_decay=p.weight_decay)
+
+trainset = StructuresDataset(root='./datasets/full_train_ds/',
+                             pre_transform=Compose((FaceAttributes(), NodeCurvature(),
+                                                    FaceToEdge(), TwoHop())))
+
+samples = len(trainset)
+cutoff = int(np.floor(samples*(1-p.validation_split)))
+train_indices = torch.tensor([i for i in range(0, cutoff)])
+train = trainset[train_indices]
+
+validset = trainset[cutoff:]
+trainset = trainset[:cutoff]
+
+sorted(glob.glob('./datasets/full_train_ds/processed/data_*.pt'))