Bug fixes and doc tweaks for fw 2.2 release

* Fix issues with new pcap port guessing logic
* Bump versions for release

CHANGELOG.rst

@@ -3,9 +3,11 @@ Changelog
 =========
 
 
-[unreleased]
+[20220107]
 ============
 
+* add support for arm64 macos and linux. Releases are now built and tested on these platforms
+* add support for Python 3.10
 * update supported vcpkg tag to 2021.05.12
 * add preliminary cpack and install support. It should be possible to use a pre-built SDK package
   instead of including the SDK in the build tree of your project
@@ -73,7 +75,7 @@ ouster_viz
 
 python
 ------
-* update ouster-sdk version to 0.3.0b3
+* update ouster-sdk version to 0.3.0
 * improve heuristics for identifying sensor data in pcaps, including new packet formats
 * release builds for wheels on Windows now use the VS 2017 toolchain and runtime (previously 2019)
 * fix potential use-after-free in ``LidarScan.fields``

CMakeLists.txt

@@ -4,7 +4,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 include(DefaultBuildType)
 
 # ==== Project Name ====
-project(ouster_example VERSION 20210608)
+project(ouster_example VERSION 20220107)
 
 # ==== Options ====
 option(CMAKE_POSITION_INDEPENDENT_CODE "Build position independent code." ON)

README.rst

@@ -146,7 +146,6 @@ Keyboard controls:
     ``p``         Increase point size
     ``o``         Decrease point size
     ``m``         Cycle point cloud coloring mode
-    ``v``         Toggle range cycling
     ``b``         Cycle top 2D image
     ``n``         Cycle bottom 2D image
     ``shift + r`` Reset camera

ouster_client/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.1.0)
 
 # ==== Project Name ====
 project(ouster_client VERSION 0.3.0)
-set(ouster_client_VERSION_SUFFIX "b3")
+set(ouster_client_VERSION_SUFFIX "")
 
 # ==== Requirements ====
 find_package(Eigen3 REQUIRED)

python/README.rst

@@ -25,13 +25,13 @@ Pre-built binaries are provided on `PyPI`_ for the following platforms:
 - Most glibc-based Linux distributions on x86_64 and ARM64 platforms (``manylinux2010_x86_64``,
   ``manylinux2014_aarch64``)
 - macOS >= 10.13 on x86_64 platforms (``macosx_10_13_x86_64``)
-- macOS >= 11.0 on Apple M1 for Python >= 3.8(``macosx_11_0_arm64``)
+- macOS >= 11.0 on Apple M1 for Python >= 3.8 (``macosx_11_0_arm64``)
 - Windows 10 on x86_64 platforms (``win_amd64``)
 
 Building from source is supported on:
 
-- Ubuntu 18.04, 20.04, and Debian 10 (x86-64, AArch64)
-- macOS >= 10.13 (x86-64)
+- Ubuntu 18.04, 20.04, and Debian 10 (x86-64, aarch64)
+- macOS >= 10.13 (x86-64), >= 11.0 (arm64)
 - Windows 10 (x86-64)
 
 .. _PyPI: https://pypi.org/project/ouster-sdk/

python/docs/conf.py

@@ -25,7 +25,7 @@
 
 # The full version, including alpha/beta/rc tags
 version = '0.3'
-release = '0.3.0b1'
+release = '0.3.0'
 
 
 # -- General configuration ---------------------------------------------------
@@ -120,7 +120,7 @@
 # napoleon_use_param = False
 
 # ----- Todos Configs ------
-todo_include_todos = True
+todo_include_todos = False
 todo_link_only = True
 todo_emit_warnings = True
 

python/docs/devel.rst

@@ -14,6 +14,8 @@ Building the Python SDK from source requires several dependencies:
 - `jsoncpp <https://github.com/open-source-parsers/jsoncpp>`_ >= 1.7
 - `libtins <http://libtins.github.io/>`_ >= 3.4
 - `libpcap <https://www.tcpdump.org/>`_
+- `libglfw3 <https://www.glfw.org/>`_ >= 3.2
+- `libglew <http://glew.sourceforge.net/>`_ >= 2.1
 - `Python <https://www.python.org/>`_ >= 3.6 (with headers and development libraries)
 - `pybind11 <https://pybind11.readthedocs.io>`_ >= 2.0
 
@@ -42,6 +44,9 @@ After you have the system dependencies, you can build the SDK with:
    # first, specify the path to the ouster_example repository
    $ export OUSTER_SDK_PATH=<PATH TO OUSTER_EXAMPLE REPO>
 
+   # make sure you have an up-to-date version of pip installed
+   $ python3 -m pip install --user --upgrade pip
+
    # then, build an installable "wheel" package
    $ python3 -m pip wheel --no-deps $OUSTER_SDK_PATH/python
 
@@ -64,23 +69,24 @@ The currently tested vcpkg tag is ``2021.05.12``. After that, using a developer
 .. code:: powershell
 
    # first, specify the path to the ouster_example repository
-   PS > $env:OUSTER_SDK_PATH=<PATH TO OUSTER_EXAMPLE>
+   PS > $env:OUSTER_SDK_PATH="<PATH TO OUSTER_EXAMPLE>"
 
-   # point cmake to the location of vcpkg
-   PS > $env:CMAKE_TOOLCHAIN_FILE=<PATH TO VCPKG REPO>/scripts/buildsystems/vcpkg.cmake
+   # point cmake to the location of vcpkg (make sure to use an absolute path)
+   PS > $env:CMAKE_TOOLCHAIN_FILE="<PATH TO VCPKG REPO>\scripts\buildsystems\vcpkg.cmake"
 
    # then, build an installable "wheel" package
-   PS > py -m pip wheel --no-deps $env:OUSTER_SDK_PATH\python
+   PS > py -m pip wheel --no-deps "$env:OUSTER_SDK_PATH\python"
 
    # or just install directly (virtualenv recommended)
-   PS > py -m pip install $env:OUSTER_SDK_PATH\python
+   PS > py -m pip install "$env:OUSTER_SDK_PATH\python"
 
 See the top-level README in the `Ouster Example repository`_ for more details on setting up a
 development environment on Windows.
 
 .. _vcpkg: https://github.com/microsoft/vcpkg/blob/master/README.md
 .. _Ouster Example repository: https://github.com/ouster-lidar/ouster_example
 
+
 Developing
 ==========
 
@@ -95,11 +101,11 @@ The Ouster SDK package includes configuration for ``flake8`` and ``mypy``. To ru
    # install and run flake8 linter
    $ python3 -m pip install flake8
    $ cd ${OUSTER_SDK_PATH}/python
-   $ flake8
+   $ python3 -m flake8
 
    # install and run mypy in an environment with
    $ python3 -m pip install mypy
-   $ mypy src/
+   $ python3 -m mypy src/
 
 
 Running Tests
@@ -111,14 +117,14 @@ SDK package:
 .. code:: console
 
    $ cd ${OUSTER_SDK_PATH}/python
-   $ pytest
+   $ python3 -m pytest
 
 To run tests against multiple Python versions simultaneously, use the ``tox`` package:
 
 .. code:: console
 
    $ cd ${OUSTER_SDK_PATH}/python
-   $ tox
+   $ python3 -m tox
 
 This will take longer, since it will build the package from a source distribution for each supported
 Python version available.

python/docs/examples.rst

@@ -546,7 +546,7 @@ The visualizer can be controlled with mouse and keyboard:
 
 .. include:: ../../README.rst
     :start-line: 136
-    :end-line: 167
+    :end-line: 169
 
 To run the visualizer with a sensor:
 

python/docs/quickstart.rst

@@ -187,6 +187,7 @@ Just like with the sample data, you can create a :py:class:`.PacketSource` from
 .. code:: python
 
    >>> source = client.Sensor(hostname)
+   >>> info = source.metadata
 
 Now we have a ``source`` from our sensor! To visualize data from your sensor, proceed to
 `Visualizing Lidar Data`_ directly below.

python/setup.py

@@ -119,7 +119,7 @@ def run(self):
 setup(
     name='ouster-sdk',
     url='https://github.com/ouster-lidar/ouster_example',
-    version='0.3.0b3',
+    version='0.3.0',
     package_dir={'': 'src'},
     packages=find_namespace_packages(where='src'),
     namespace_packages=['ouster'],

python/src/ouster/pcap/pcap.py

@@ -33,6 +33,7 @@ class _UDPStreamInfo:
 
 class _stream_info:
     """Gather some useful info about UDP data in a pcap."""
+
     def __init__(self, infos: Iterable[_pcap.packet_info]) -> None:
         self.total_packets = 0
         self.encapsulation_protocol = set()
@@ -60,51 +61,53 @@ def _guess_ports(udp_streams: Dict[_UDPStreamKey, _UDPStreamInfo],
     """Find possible UDP sources matching the metadata.
 
     The current approach is roughly: 1) treat each unique source / destination
-    port and IP as a single logical 'stream' of data. 2) filter out streams that
-    don't match the expected destination ports and packet sizes specified in the
-    metadata. 3) produce pairs of lidar/imu streams that have matching source
-    IPs, or None if a corresponding stream can't be found.
+    port and IP as a single logical 'stream' of data, 2) filter out streams that
+    don't match the expected packet sizes specified by the metadata, 3) pair up
+    any potential lidar/imu streams that appear to be coming from the same
+    sensor (have matching source IPs) 4) and finally, filter out the pairs that
+    contradict any ports specified in the metadata.
 
     Returns:
         List of dst port pairs that probably contain lidar/imu data. Duplicate
         entries are possible and indicate packets from distinct sources.
     """
 
-    # yapf: disable
-    # ports are zero if not specified (e.g. in older metadata)
-    def filter_keys(size: int, dst_port: int) -> Set[Optional[_UDPStreamKey]]:
-        return {k for k, v in udp_streams.items()
-                if (size in v.payload_size)
-                and (not dst_port or k.dst_port == dst_port)}
+    # allow lone streams when there's no matching data from the same ip
+    lidar_keys: Set[Optional[_UDPStreamKey]] = {None}
+    imu_keys: Set[Optional[_UDPStreamKey]] = {None}
 
-    # find all lidar and imu 'streams' that match metadata
+    # find all lidar and imu 'streams' that match expected packet sizes
     pf = _client.PacketFormat.from_info(info)
-    lidar_keys = filter_keys(pf.lidar_packet_size, info.udp_port_lidar)
-    imu_keys = filter_keys(pf.imu_packet_size, info.udp_port_imu)
+    ss = udp_streams.items()
+    lidar_keys |= {k for k, v in ss if pf.lidar_packet_size in v.payload_size}
+    imu_keys |= {k for k, v in ss if pf.imu_packet_size in v.payload_size}
 
     # find all src ips for candidate streams
     lidar_src_ips = {k.src_ip for k in lidar_keys if k}
     imu_src_ips = {k.src_ip for k in imu_keys if k}
 
-    # allow lone streams when there's no matching data from the same ip
-    lidar_keys.add(None)
-    imu_keys.add(None)
-
-    # full join on matching src ip to produce distinct lidar/imu stream choices
-    keys = [(kl, ki) for kl in lidar_keys for ki in imu_keys
-            if (not kl and ki and ki.src_ip not in lidar_src_ips)
-            or (kl and not ki and kl.src_ip not in imu_src_ips)
-            or (kl and ki and kl.src_ip == ki.src_ip)]
-    # yapf: enable
+    # yapf: disable
+    # "full outer join" on src_ip to produce lidar/imu streams from one source
+    keys = [(klidar, kimu) for klidar in lidar_keys for kimu in imu_keys
+            if (klidar and kimu and klidar.src_ip == kimu.src_ip)
+            or (not klidar and kimu and kimu.src_ip not in lidar_src_ips)
+            or (klidar and not kimu and klidar.src_ip not in imu_src_ips)]
 
     # map down to just dst port pairs, with 0 meaning none found
-    ports = [(kl.dst_port if kl else 0, ki.dst_port if ki else 0)
-             for (kl, ki) in keys]
+    ports = [(klidar.dst_port if klidar else 0, kimu.dst_port if kimu else 0)
+             for (klidar, kimu) in keys]
+
+    # filter out candidates that don't match specified ports
+    lidar_spec, imu_spec = info.udp_port_lidar, info.udp_port_imu
+    guesses = [(plidar, pimu) for plidar, pimu in ports
+               if (plidar == lidar_spec or lidar_spec == 0 or plidar == 0)
+               and (pimu == imu_spec or imu_spec == 0 or pimu == 0)]
+    # yapf: enable
 
     # sort sensor ports to prefer both found > just lidar > just imu
-    ports.sort(reverse=True, key=lambda p: (p[0] != 0, p[1] != 0, p))
+    guesses.sort(reverse=True, key=lambda p: (p[0] != 0, p[1] != 0, p))
 
-    return ports
+    return guesses
 
 
 def _packet_info_stream(path: str) -> Iterator[_pcap.packet_info]:
@@ -147,8 +150,7 @@ def __init__(self,
         When not specified, ports are guessed by sampling some packets and
         looking for the expected packet size based on the sensor metadata. If
         packets that might be valid sensor data appear on multiple ports, one is
-        chosen arbitrarily. See ``_guess_streams`` for details. on the
-        heuristics.
+        chosen arbitrarily. See ``_guess_ports`` for details. on the heuristics.
 
         Packets with the selected destination port that clearly don't match the
         metadata (e.g. wrong size or init_id) will be silently ignored.
@@ -178,13 +180,12 @@ def __init__(self,
         stats = _stream_info(islice(_packet_info_stream(pcap_path), n_packets))
         self._guesses = _guess_ports(stats.udp_streams, self._metadata)
 
-        # if ports were inferred, they must be equal or more specific
+        # fill in unspecified (0) ports with inferred values
         if len(self._guesses) > 0:
             lidar_guess, imu_guess = self._guesses[0]
-            assert lidar_port == lidar_guess or lidar_port <= 0
-            assert imu_port == imu_guess or imu_port <= 0
-            self._metadata.udp_port_lidar = lidar_guess
-            self._metadata.udp_port_imu = imu_guess
+            # guess != port only if port == 0 or guess == 0
+            self._metadata.udp_port_lidar = lidar_guess or lidar_port
+            self._metadata.udp_port_imu = imu_guess or imu_port
 
         self._rate = rate
         self._handle = _pcap.replay_initialize(pcap_path)

python/tests/conftest.py

@@ -1,5 +1,6 @@
 from contextlib import closing
 from os import path
+from typing import Iterator
 
 import pytest
 
@@ -42,20 +43,41 @@ def meta(base_name: str):
 
 
 @pytest.fixture
-def packet(base_name: str, meta: client.SensorInfo) -> client.LidarPacket:
-    pcap_path = path.join(DATA_DIR, f"{base_name}.pcap")
-    with closing(pcap.Pcap(pcap_path, meta)) as source:
-        for p in source:
+def meta_2_0():
+    meta_path = path.join(DATA_DIR, f"{TESTS['legacy-2.0']}.json")
+    with open(meta_path, 'r') as f:
+        return client.SensorInfo(f.read())
+
+
+@pytest.fixture
+def real_pcap_path(base_name: str, meta: client.SensorInfo) -> str:
+    return path.join(DATA_DIR, f"{base_name}.pcap")
+
+
+@pytest.fixture
+def real_pcap(real_pcap_path: str,
+              meta: client.SensorInfo) -> Iterator[pcap.Pcap]:
+    pcap_obj = pcap.Pcap(real_pcap_path, meta)
+    yield pcap_obj
+    pcap_obj.close()
+
+
+@pytest.fixture
+def packet(real_pcap_path: str, meta: client.SensorInfo) -> client.LidarPacket:
+    # note: don't want to depend on the pcap fixture, since this consumes the
+    # iterator and it can be shared
+    with closing(pcap.Pcap(real_pcap_path, meta)) as real_pcap:
+        for p in real_pcap:
             if isinstance(p, client.LidarPacket):
                 return p
-        raise RuntimeError("Failed to find lidar packet in text fixture")
+        raise RuntimeError("Failed to find lidar packet in test fixture")
 
 
 @pytest.fixture
-def packets(base_name: str, meta: client.SensorInfo) -> client.PacketSource:
-    pcap_path = path.join(DATA_DIR, f"{base_name}.pcap")
-    with closing(pcap.Pcap(pcap_path, meta)) as source:
-        ps = list(source)
+def packets(real_pcap_path: str,
+            meta: client.SensorInfo) -> client.PacketSource:
+    with closing(pcap.Pcap(real_pcap_path, meta)) as real_pcap:
+        ps = list(real_pcap)
         return client.Packets(ps, meta)