Merge pull request #290 from scikit-hep/cmsnano-profile

coordinated CMS NanoAOD release

.travis.yml

@@ -59,9 +59,9 @@ install:
   - pip install --upgrade setuptools-scm
   - pip install $NPY
   - python -c 'import numpy; print(numpy.__version__)'
-  - pip install "awkward>=0.10.0"
+  - pip install "awkward>=0.11.0rc1"
   - python -c 'import awkward; print(awkward.__version__)'
-  - pip install "uproot-methods>=0.6.0"
+  - pip install "uproot-methods>=0.7.0rc1"
   - python -c 'import uproot_methods; print(uproot_methods.__version__)'
   - pip install cachetools pkgconfig lz4 xxhash mock requests "pytest>=3.9" pytest-runner
   - pip install lz4 requests

README.rst

@@ -61,8 +61,8 @@ Strict dependencies:
 --------------------
 
 - `numpy <https://scipy.org/install.html>`__ (1.13.1+)
-- `awkward-array <https://github.com/scikit-hep/awkward-array>`__ (0.10.0+)
-- `uproot-methods <https://github.com/scikit-hep/uproot-methods>`__ (0.6.0+)
+- `awkward-array <https://github.com/scikit-hep/awkward-array>`__ (0.11.0+)
+- `uproot-methods <https://github.com/scikit-hep/uproot-methods>`__ (0.7.0+)
 - `cachetools <https://pypi.org/project/cachetools>`__
 
 Optional dependencies:

appveyor.yml

@@ -46,9 +46,9 @@ install:
 build_script:
   - "python -m pip install --upgrade pip"
   - "pip install %NUMPY%"
-  - "pip install \"awkward>=0.10.0\""
+  - "pip install \"awkward>=0.11.0rc1\""
   - "python -c \"import awkward; print(awkward.__version__)\""
-  - "pip install \"uproot-methods>=0.6.0\""
+  - "pip install \"uproot-methods>=0.7.0rc1\""
   - "python -c \"import uproot_methods; print(uproot_methods.__version__)\""
   - "pip install cachetools"
   - "pip install -i https://pypi.anaconda.org/carlkl/simple backports.lzma"

binder/requirements.txt

@@ -1,6 +1,6 @@
 numpy>=1.13.1
-awkward>=0.10.0
-uproot-methods>=0.6.0
+awkward>=0.11.0rc1
+uproot-methods>=0.7.0rc1
 cachetools
 pkgconfig
 lz4

requirements.txt

@@ -1,5 +1,5 @@
 numpy>=1.13.1
-awkward>=0.10.0
-uproot-methods>=0.6.0
+awkward>=0.11.0rc1
+uproot-methods>=0.7.0rc1
 cachetools
 backports.lzma;python_version<"3.3"

setup.py

@@ -141,7 +141,7 @@ def get_description():
       download_url = "https://github.com/scikit-hep/uproot/releases",
       license = "BSD 3-clause",
       test_suite = "tests",
-      install_requires = ["numpy>=1.13.1", "awkward>=0.10.0", "uproot-methods>=0.6.0", "cachetools"],
+      install_requires = ["numpy>=1.13.1", "awkward>=0.11.0rc1", "uproot-methods>=0.7.0rc1", "cachetools"],
       setup_requires = ["pytest-runner"],
       tests_require = ["pytest>=3.9", "pkgconfig", "lz4", 'backports.lzma;python_version<"3.3"', "xxhash", "mock", "requests"],
       classifiers = [

uproot/interp/interp.py

@@ -10,6 +10,8 @@ class Interpretation(object):
 
     awkward = awkward
 
+    debug_reading = False
+
     def awkwardlib(self, lib):
         cls = type(self)
         out = cls.__new__(cls)

uproot/interp/jagged.py

@@ -131,4 +131,6 @@ def finalize(self, destination, branch):
 
         out = self.awkward.Methods.maybemixin(type(content), self.awkward.JaggedArray).fromcounts(destination.counts, content)
         out.leafcount = leafcount
+        if self.debug_reading:
+            print("reading {0}".format(repr(out)))
         return out

uproot/interp/numerical.py

@@ -72,6 +72,8 @@ def clip(self, destination, itemstart, itemstop, entrystart, entrystop):
         #     return destination[entrystart:entrystop]
 
     def finalize(self, destination, branch):
+        if self.debug_reading:
+            print("reading {0}".format(repr(destination)))
         return destination
 
 class asdtype(_asnumeric):
@@ -194,7 +196,10 @@ def clip(self, destination, itemstart, itemstop, entrystart, entrystop):
 
     def finalize(self, destination, branch):
         array, stop = destination
-        return array[:stop]
+        out = array[:stop]
+        if self.debug_reading:
+            print("reading {0}".format(repr(out)))
+        return out
 
 class asdouble32(_asnumeric):
     # makes __doc__ attribute mutable before Python 3.3
@@ -365,4 +370,6 @@ def clip(self, destination, itemstart, itemstop, entrystart, entrystop):
         return destination[itemstart:itemstop]
 
     def finalize(self, destination, branch):
+        if self.debug_reading:
+            print("reading {0}".format(repr(destination)))
         return destination

uproot/interp/objects.py

@@ -171,7 +171,10 @@ def clip(self, destination, itemstart, itemstop, entrystart, entrystop):
         return self.content.clip(destination, itemstart, itemstop, entrystart, entrystop)
 
     def finalize(self, destination, branch):
-        return self.awkward.Table.fromrec(self.content.finalize(destination, branch))
+        out = self.awkward.Table.fromrec(self.content.finalize(destination, branch))
+        if self.debug_reading:
+            print("reading {0}".format(repr(out)))
+        return out
 
 class asobj(uproot.interp.interp.Interpretation):
     # makes __doc__ attribute mutable before Python 3.3
@@ -222,12 +225,14 @@ def clip(self, destination, itemstart, itemstop, entrystart, entrystop):
 
     def finalize(self, destination, branch):
         if self.cls._arraymethods is None:
-            return self.awkward.ObjectArray(self.content.finalize(destination, branch), self.cls._fromrow)
+            out = self.awkward.ObjectArray(self.content.finalize(destination, branch), self.cls._fromrow)
         else:
             cls = self.awkward.Methods.mixin(self.cls._arraymethods, self.awkward.ObjectArray)
             out = cls.__new__(cls)
             out._initObjectArray(self.content.finalize(destination, branch))
-            return out
+        if self.debug_reading:
+            print("reading {0}".format(repr(out)))
+        return out
 
 class _variable(uproot.interp.interp.Interpretation):
     def __init__(self, content, generator, *args, **kwargs):
@@ -272,7 +277,10 @@ def clip(self, destination, itemstart, itemstop, entrystart, entrystop):
         return self.content.clip(destination, itemstart, itemstop, entrystart, entrystop)
 
     def finalize(self, destination, branch):
-        return self.awkward.ObjectArray(self.content.finalize(destination, branch), self.generator, *self.args, **self.kwargs)
+        out = self.awkward.ObjectArray(self.content.finalize(destination, branch), self.generator, *self.args, **self.kwargs)
+        if self.debug_reading:
+            print("reading {0}".format(repr(out)))
+        return out
 
 class asgenobj(_variable):
     # makes __doc__ attribute mutable before Python 3.3

uproot/tree.py

@@ -544,35 +544,32 @@ def lazyarray(self, branch, interpretation=None, entrysteps=None, entrystart=Non
 
     def lazyarrays(self, branches=None, namedecode="utf-8", entrysteps=None, entrystart=None, entrystop=None, flatten=False, profile=None, awkwardlib=None, cache=None, basketcache=None, keycache=None, executor=None, persistvirtual=False):
         entrystart, entrystop = _normalize_entrystartstop(self.numentries, entrystart, entrystop)
-        entrysteps = self._normalize_entrysteps(entrysteps, branches, entrystart, entrystop, keycache)
+        entrysteps = list(self._normalize_entrysteps(entrysteps, branches, entrystart, entrystop, keycache))
         awkward = _normalize_awkwardlib(awkwardlib)
         branches = list(self._normalize_branches(branches, awkward))
 
         lazytree = _LazyTree(self._context.sourcepath, self._context.treename, self, dict((b.name, x) for b, x in branches), flatten, awkward.__name__, basketcache, keycache, executor)
 
-        chunks = []
-        counts = []
-        rowstart = 0
-        for start, stop in entrysteps:
-            numentries = stop - start
-            chunks.append(awkward.Table())
-            for branch, interpretation in branches:
-                inner = interpretation
-                while isinstance(inner, asjagged):
-                    inner = inner.content
-                if isinstance(inner, asobj) and getattr(inner.cls, "_arraymethods", None) is not None:
-                    virtualarray = awkward.Methods.mixin(inner.cls._arraymethods, awkward.VirtualArray)
-                elif isinstance(inner, asgenobj) and getattr(inner.generator.cls, "_arraymethods", None) is not None:
-                    virtualarray = awkward.Methods.mixin(inner.generator.cls._arraymethods, awkward.VirtualArray)
-                else:
-                    virtualarray = awkward.VirtualArray
-                name = branch.name if namedecode is None else branch.name.decode(namedecode)
-                chunks[-1][name] = virtualarray(lazytree, (branch.name, start, stop), cache=cache, type=awkward.type.ArrayType(numentries, interpretation.type), persistvirtual=persistvirtual)
-            chunks[-1].rowstart = rowstart
-            rowstart += numentries
-            counts.append(numentries)
+        out = awkward.Table()
+        for branch, interpretation in branches:
+            inner = interpretation
+            while isinstance(inner, asjagged):
+                inner = inner.content
+            if isinstance(inner, asobj) and getattr(inner.cls, "_arraymethods", None) is not None:
+                VirtualArray = awkward.Methods.mixin(inner.cls._arraymethods, awkward.VirtualArray)
+            elif isinstance(inner, asgenobj) and getattr(inner.generator.cls, "_arraymethods", None) is not None:
+                VirtualArray = awkward.Methods.mixin(inner.generator.cls._arraymethods, awkward.VirtualArray)
+            else:
+                VirtualArray = awkward.VirtualArray
+
+            chunks = []
+            counts = []
+            for start, stop in entrysteps:
+                chunks.append(VirtualArray(lazytree, (branch.name, start, stop), cache=cache, type=awkward.type.ArrayType(stop - start, interpretation.type), persistvirtual=persistvirtual))
+                counts.append(stop - start)
+            name = branch.name.decode("ascii") if namedecode is None else branch.name.decode(namedecode)
+            out[name] = awkward.ChunkedArray(chunks, counts)
 
-        out = awkward.ChunkedArray(chunks, counts)
         if profile is not None:
             out = uproot_methods.profiles.transformer(profile)(out)
         return out
@@ -1573,13 +1570,13 @@ def lazyarray(self, interpretation=None, entrysteps=None, entrystart=None, entry
         while isinstance(inner, asjagged):
             inner = inner.content
         if isinstance(inner, asobj) and getattr(inner.cls, "_arraymethods", None) is not None:
-            virtualarray = awkward.Methods.mixin(inner.cls._arraymethods, awkward.VirtualArray)
+            VirtualArray = awkward.Methods.mixin(inner.cls._arraymethods, awkward.VirtualArray)
             chunkedarray = awkward.Methods.mixin(inner.cls._arraymethods, awkward.ChunkedArray)
         elif isinstance(inner, asgenobj) and getattr(inner.generator.cls, "_arraymethods", None) is not None:
-            virtualarray = awkward.Methods.mixin(inner.generator.cls._arraymethods, awkward.VirtualArray)
+            VirtualArray = awkward.Methods.mixin(inner.generator.cls._arraymethods, awkward.VirtualArray)
             chunkedarray = awkward.Methods.mixin(inner.generator.cls._arraymethods, awkward.ChunkedArray)
         else:
-            virtualarray = awkward.VirtualArray
+            VirtualArray = awkward.VirtualArray
             chunkedarray = awkward.ChunkedArray
 
         lazybranch = _LazyBranch(self._context.sourcepath, self._context.treename, self.name, self, interpretation, flatten, awkward.__name__, basketcache, keycache, executor)
@@ -1588,7 +1585,7 @@ def lazyarray(self, interpretation=None, entrysteps=None, entrystart=None, entry
         counts = []
         for start, stop in entrysteps:
             numentries = stop - start
-            chunks.append(virtualarray(lazybranch, (start, stop), cache=cache, type=awkward.type.ArrayType(numentries, interpretation.type), persistvirtual=persistvirtual))
+            chunks.append(VirtualArray(lazybranch, (start, stop), cache=cache, type=awkward.type.ArrayType(numentries, interpretation.type), persistvirtual=persistvirtual))
             counts.append(numentries)
 
         return chunkedarray(chunks, counts)
@@ -1929,43 +1926,40 @@ def lazyarrays(path, treepath, branches=None, namedecode="utf-8", entrysteps=flo
 
     path2count = numentries(path, treepath, total=False, localsource=localsource, xrootdsource=xrootdsource, httpsource=httpsource, executor=executor, blocking=True)
 
-    listbranches = None
+    lazyfiles = _LazyFiles(paths, treepath, branches, entrysteps, flatten, awkward.__name__, basketcache, keycache, executor, persistvirtual, localsource, xrootdsource, httpsource, options)
+
     for path in paths:
         file = uproot.rootio.open(path, localsource=localsource, xrootdsource=xrootdsource, httpsource=httpsource, **options)
         try:
             tree = file[treepath]
         except KeyError:
             continue
-        listbranches = list(tree._normalize_branches(branches, awkward))
+        branches = list(tree._normalize_branches(branches, awkward))
         break
 
-    if listbranches is None:
+    if branches is None:
         raise ValueError("no matching paths contained a tree named {0}".format(repr(treepath)))
 
-    lazyfiles = _LazyFiles(paths, treepath, branches, entrysteps, flatten, awkward.__name__, basketcache, keycache, executor, persistvirtual, localsource, xrootdsource, httpsource, options)
+    out = awkward.Table()
+    for branch, interpretation in branches:
+        inner = interpretation
+        while isinstance(inner, asjagged):
+            inner = inner.content
+        if isinstance(inner, asobj) and getattr(inner.cls, "_arraymethods", None) is not None:
+            VirtualArray = awkward.Methods.mixin(inner.cls._arraymethods, awkward.VirtualArray)
+        elif isinstance(inner, asgenobj) and getattr(inner.generator.cls, "_arraymethods", None) is not None:
+            VirtualArray = awkward.Methods.mixin(inner.generator.cls._arraymethods, awkward.VirtualArray)
+        else:
+            VirtualArray = awkward.VirtualArray
+
+        chunks = []
+        counts = []
+        for pathi, path in enumerate(paths):
+            chunks.append(VirtualArray(lazyfiles, (pathi, branch.name), cache=cache, type=awkward.type.ArrayType(path2count[path], interpretation.type), persistvirtual=persistvirtual))
+            counts.append(path2count[path])
+        name = branch.name.decode("ascii") if namedecode is None else branch.name.decode(namedecode)
+        out[name] = awkward.ChunkedArray(chunks, counts)
 
-    chunks = []
-    counts = []
-    rowstart = 0
-    for pathi, path in enumerate(paths):
-        chunks.append(awkward.Table())
-        for branch, interpretation in listbranches:
-            inner = interpretation
-            while isinstance(inner, asjagged):
-                inner = inner.content
-            if isinstance(inner, asobj) and getattr(inner.cls, "_arraymethods", None) is not None:
-                virtualarray = awkward.Methods.mixin(inner.cls._arraymethods, awkward.VirtualArray)
-            elif isinstance(inner, asgenobj) and getattr(inner.generator.cls, "_arraymethods", None) is not None:
-                virtualarray = awkward.Methods.mixin(inner.generator.cls._arraymethods, awkward.VirtualArray)
-            else:
-                virtualarray = awkward.VirtualArray
-            name = branch.name if namedecode is None else branch.name.decode(namedecode)
-            chunks[-1][name] = virtualarray(lazyfiles, (pathi, branch.name), cache=cache, type=awkward.type.ArrayType(path2count[path], interpretation.type), persistvirtual=persistvirtual)
-        chunks[-1].rowstart = rowstart
-        rowstart += path2count[path]
-        counts.append(path2count[path])
-
-    out = awkward.ChunkedArray(chunks, counts)
     if profile is not None:
         out = uproot_methods.profiles.transformer(profile)(out)
     return out

uproot/version.py

@@ -4,7 +4,7 @@
 
 import re
 
-__version__ = "3.6.5"
+__version__ = "3.7.0rc1"
 version = __version__
 version_info = tuple(re.split(r"[-\.]", __version__))