support big-endian machines (#3361)

Summary:
Pull Request resolved: #3361

Fix a few issues in the PR.
Normally all tests should pass on a litlle-endian machine

Reviewed By: junjieqi

Differential Revision: D56003181

fbshipit-source-id: 405dec8c71898494f5ddcd2718c35708a1abf9cb

contrib/vecs_io.py

@@ -14,8 +14,8 @@
 
 def ivecs_read(fname):
     a = np.fromfile(fname, dtype='int32')
-    if sys.byteorder == 'big':
-      a.byteswap(inplace=True)
+    if sys.big_endian:
+        a.byteswap(inplace=True)
     d = a[0]
     return a.reshape(-1, d + 1)[:, 1:].copy()
 
@@ -25,6 +25,7 @@ def fvecs_read(fname):
 
 
 def ivecs_mmap(fname):
+    assert not sys.big_endian
     a = np.memmap(fname, dtype='int32', mode='r')
     d = a[0]
     return a.reshape(-1, d + 1)[:, 1:]
@@ -36,7 +37,11 @@ def fvecs_mmap(fname):
 
 def bvecs_mmap(fname):
     x = np.memmap(fname, dtype='uint8', mode='r')
-    d = x[:4].view('int32')[0]
+    if sys.big_endian:
+        da = x[:4][::-1].copy()
+        d = da.view('int32')[0]
+    else:
+        d = x[:4].view('int32')[0]
     return x.reshape(-1, d + 4)[:, 4:]
 
 
@@ -45,6 +50,8 @@ def ivecs_write(fname, m):
     m1 = np.empty((n, d + 1), dtype='int32')
     m1[:, 0] = d
     m1[:, 1:] = m
+    if sys.big_endian:
+        m1.byteswap(inplace=True)
     m1.tofile(fname)