Hard transition to Python3 as a squashed merge (#231)

* futurize stage1, tests pass

* Catching two exceptions

* futurize stage2, tests fail

* Took out python2 wrappers, excessive casts, and fixed small things along the way

* Some old indentation errors

* Adapted to API change in configparser

* Fixed programmatic imports

* Removed explicit checks for iteritems()

* Adapted to api change in socket

* Removed all remaining python2 wrappers

* avoiding configparser DuplicationErrors more generally

* pickle doesn't do local classes like cPickle did

* Missed a few old_div references

* Found some remaining subscripting of iterators

* Fixed two PtyScan classes - you have to be careful with /= on arrays in python 3

* Fixed the live plotter

* warn() method deprecated in logging

* imp module is deprecated

* Enough with the 'Failed to register' warnings

* Removed pickling unsupported data types from h5io

* Update travis.yaml, full_dependencies and core dependencies for python 3
`

* fixes saving of np.recarrays by casting to fixed length string before saving.

* Reinstates pickling of unknown types.

* Update doc conf to python3

* Refactor of script2rst.py gets rid of 1800 or so errors.

* Corrects for new lines in the middle of a sentence in .help.

* The .scan field doesn't exist anymore

* converts ptypy.inspect to python3

* Move __idiv__ --> __truediv__

* Needs an escape character

* extra line here

* removes invalid doc strings. The class doc string already contains the necessary parameters anyway

* removes invalid, duplicated docstring.

* fixes test that fails because I changed to __truediv__. It would have been good if this had failed on implementation rather than checking it was there.

* backslashes aren't needed here

* Turns autoplot on for this.

* add path to the data

* The cxro lookup was a victim of the python3 convertor. Works now.

* This isn't an integer in p3. It needs to be

* Defaults tree has moved

* Need to check high and low limits properly for iterables too

* Needs to be u.Param

* Allows backwards compatibility. We can now h5read files that were saved in the python2.7 version of ptypy

* removed record array cause it was not used as such. Now it's a simple structured array where unicode is replaced with byte strings. h5py can not handle numpy U dtypes in arrays

* Bumped to version 0.4.0 in preparation for merge

* Updated release notes

* bringing the doc up to right version automatically

.gitignore

@@ -11,6 +11,7 @@ doc/rst/data_management.rst
 doc/_img/*.png
 tutorial/*.png
 ghostdriver*
+doc/version.py
 ptypy/version.py
 .idea/
 .cache/

.travis.yml

@@ -2,9 +2,9 @@ cache: apt
 sudo: true
 language: python
 python:
-  - 2.7
+  - 3.7
 before_install:
-  - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; # grab miniconda
+  - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; # grab miniconda
   - bash miniconda.sh -b -p $HOME/miniconda # install miniconda
   - export PATH="$HOME/miniconda/bin:$PATH" # add it to the path
   - conda config --set always_yes yes --set changeps1 no # we want it to always do yes, and start a new console
@@ -19,8 +19,8 @@ env:
 install: True
 
 script:
-  - conda env create --file ${TEST_ENV_NAME}.yml #set up the environment thats in the file
-  - source activate ${TEST_ENV_NAME} # activate it
+  - conda env create --file ${TEST_ENV_NAME}.yml;
+  - source activate ${TEST_ENV_NAME}; # activate it
   - conda install pytest # additional dependencies for the tests
   - pip install pytest-cov
   - pip install coveralls
@@ -32,5 +32,4 @@ script:
 after_script:
   - coveralls
 
-allow_failures:
-- python: "3.3"
+

benchmark/class_benchmarks.py

@@ -35,15 +35,15 @@ def add_views(nviews):
 for k in range(int(steps)):
     add_views(nviews)
     gc.collect()
-    print k 
-    print C1._recs.values()[0].nbytes / 1e6
+    print(k) 
+    print(list(C1._recs.values())[0].nbytes / 1e6)
 
-print C1.formatted_report()
+print(C1.formatted_report())
 u.pause(1)
 gc.collect()
 u.pause(4)
 u.pause(4)
-print C1.formatted_report()
+print(C1.formatted_report())
 add_views(nviews)
 u.pause(4)
 

core_dependencies.yml

@@ -2,7 +2,7 @@ name: core_dependencies
 channels:
   - conda-forge
 dependencies:
-  - python=2.7
+  - python=3.7
   - numpy
   - scipy
   - h5py

doc/conf.py

@@ -23,9 +23,9 @@
 # generate paramters.rst and other rst
 import subprocess
 subprocess.check_call(['python', 'script2rst.py'])  # We need this to have a clean sys.argv
-subprocess.check_call(['python','parameters2rst.py'])
-subprocess.check_call(['python','tmp2rst.py'])
-execfile('version.py')
+subprocess.check_call(['python', 'parameters2rst.py'])
+subprocess.check_call(['python', 'tmp2rst.py'])
+exec(open('version.py').read())
 
 # -- General configuration ------------------------------------------------
 
@@ -64,8 +64,8 @@ def get_refs(dct, pd, depth=2, indent=''):
         if depth < 0:
             return
 
-        for k, value in dct.iteritems():
-            ref = ', see :py:data:`~%s`' % pd.children[k].entry_point if pd.children.has_key(k) else ''
+        for k, value in dct.items():
+            ref = ', see :py:data:`~%s`' % pd.children[k].entry_point if k in pd.children else ''
             if hasattr(value, 'items'):
                 v = str(value.__class__.__name__)
             elif str(value) == value:

doc/parameters2rst.py

@@ -40,7 +40,8 @@
     if is_wildcard:
         prst.write('   *Wildcard*: multiple entries with arbitrary names are accepted.\n\n')
 
-    prst.write('   '+desc.help+'\n\n')
+    # prst.write('   '+desc.help+'\n\n')
+    prst.write('   ' + desc.help.replace('<newline>', '\n').replace('\n', '\n   ') + '\n\n')
     prst.write('   '+desc.doc.replace('<newline>','\n').replace('\n', '\n   ')+'\n\n')
 
     if desc.is_symlink:

doc/script2rst.py

@@ -1,5 +1,5 @@
 import sys
-import StringIO
+import io
 import contextlib
 import os
 
@@ -12,7 +12,6 @@
 
 if len(sys.argv) == 1:
     import pkg_resources
-    import subprocess
 
     for script in scripts:
         scr = pkg_resources.resource_filename('ptypy', tutorial_dir+script)
@@ -25,11 +24,11 @@
 
 indent_keys = ['for', 'if', 'with', 'def', 'class']
 
-sout = StringIO.StringIO()
 
 
 @contextlib.contextmanager
 def stdoutIO(stdout=None):
+    sout = io.StringIO()
     old = sys.stdout
     if stdout is None:
         stdout = sout
@@ -38,12 +37,7 @@ def stdoutIO(stdout=None):
         yield stdout
     finally:
         sys.stdout = old
-
 
-def exec2str(statement):
-    with stdoutIO() as s:
-        exec(statement)
-    print(s.getvalue())
 
 script_name = sys.argv[1]
 fpy = open(script_name, 'r')
@@ -64,37 +58,10 @@ def exec2str(statement):
 """ % {'fname': os.path.split(script_name)[-1], 'this': sys.argv[0]})
 
 was_comment = True
-def debug(x):
-    print(x)
-
-def check_for_fig(wline):
-    if 'savefig' in wline:
-        print('found fig')
-        from matplotlib import pyplot as plt
-        fig = plt.gcf()
-        fig_name = name + '_%02d' % fig.number
-        fname = fig_path + fig_name + '.png'
-        plt.tight_layout()
-        fig.savefig(fname, dpi=300)
-
-        frst.write('.. figure:: '+'..'+os.sep+fname+'\n')
-        frst.write('   :width: 70 %\n')
-        frst.write('   :figclass: highlights\n')
-        other = [s.strip() for s in wline.split(';')]
-        print(other)
-        if len(other) > 1:
-            frst.write('   :name: '+other[1]+'\n\n')
-        if len(other) > 2:
-            frst.write('   '+other[2]+'\n\n')
-        frst.write('\n')
-        wline == ''
-        return True
-    else:
-        return False
 
 while True:
     line = fpy.readline()
-    if line == '':
+    if not line:
         break
     print(line)
     if 'savefig' in line:
@@ -156,22 +123,22 @@ def check_for_fig(wline):
             func += line2
             frst.write('   >>> '+line2)
             pt = fpy.tell()
-        exec func+'\n'
+        exec(func+'\n')
         continue
 
     wline = line.strip()
-    if wline == '':
+    if not wline:
         frst.write('\n')
         continue
 
     with stdoutIO() as sout:
-        exec wline
+        exec(wline)
         out = sout.getvalue()
         sout.buf = ''
+
     if len(wline) > 0:
         if line.startswith('# '):
             wline = line[2:]
-            #isfig = check_for_fig(wline)
             was_comment = True
             frst.write(wline)
         else:
@@ -183,17 +150,11 @@ def check_for_fig(wline):
             frst.write(wline+'\n')
 
         #print out
-        if out.strip() != '':
-            #frst.write('\n')
+        if out.strip():
+            print(out)
             for l in out.split('\n'):
                 frst.write(' '*3+l+'\n')
             out = ''
-    """
-    frst.write(wline+'\n')
-    if out.strip()!='':
-        frst.write('\n:Out:\n   ::\n\n')
-        for l in out.split('\n'):
-            frst.write(' '*6+l+'\n')
-    """
+
 
 

doc/version.py

@@ -12,5 +12,5 @@
     except:
         pass
     else:
-        version += git_commit.strip()
+        version += git_commit.strip().decode()
 

extra/ipynb/plotclient.ipynb

@@ -70,8 +70,8 @@
    "outputs": [],
    "source": [
     "prdict,obdict,metadict = newdata()\n",
-    "pr = prdict[prdict.keys()[0]]['data']\n",
-    "ob = obdict[prdict.keys()[0]]['data']\n",
+    "pr = prdict[list(prdict.keys())[0]]['data']\n",
+    "ob = obdict[list(prdict.keys())[0]]['data']\n",
     "fig, axes = plt.subplots(ncols=3, figsize=(12,3), dpi=100)\n",
     "\n",
     "# Plotting the object\n",

full_dependencies.yml

@@ -1,17 +1,16 @@
 name: full_dependencies
 channels:
   - conda-forge
-  - anaconda
 dependencies:
-  - python=2.7
+  - python=3.7
   - numpy
   - scipy
   - matplotlib
   - h5py
   - pyzmq
   - pep8
   - mpi4py
-  - pil
+  - pillow
   - pyfftw
   - pip
   - pip:

ptypy/__init__.py

@@ -51,17 +51,17 @@
 if not __has_zmq__:
     __zmq_msg = 'ZeroMQ not found.\nInteraction server & client disabled.\n\
     Install python-zmq via the package repositories or with `pip install --user pyzmq`'
-    verbose.logger.warn(__zmq_msg)
+    verbose.logger.warning(__zmq_msg)
 if not __has_mpi4py__:
     __mpi_msg = 'Message Passaging for Python (mpi4py) not found.\n\
     CPU-parallelization disabled.\n\
     Install python-mpi4py via the package repositories or with `pip install --user mpi4py`'
-    verbose.logger.warn(__mpi_msg)
+    verbose.logger.warning(__mpi_msg)
 if not __has_matplotlib__:
     __mpl_msg = 'Plotting for Python (matplotlib) not found.\n\
     Plotting disabled.\n\
     Install python-matplotlib via the package repositories or with `pip install --user matplotlib`'
-    verbose.logger.warn(__mpl_msg)
+    verbose.logger.warning(__mpl_msg)
 
 # Start a parameter tree
 from .utils.descriptor import EvalDescriptor