Merge pull request #435 from openxc/next

Next

.gitmodules

@@ -20,18 +20,9 @@
 [submodule "src/libs/AT-commander"]
 	path = src/libs/AT-commander
 	url = https://github.com/openxc/AT-commander
-[submodule "src/libs/nanopb"]
-	path = src/libs/nanopb
-	url = https://github.com/nanopb/nanopb.git
 [submodule "src/libs/openxc-message-format"]
 	path = src/libs/openxc-message-format
 	url = https://github.com/openxc/openxc-message-format
-[submodule "src/libs/bitfield-c"]
-	path = src/libs/bitfield-c
-	url = https://github.com/openxc/bitfield-c
-[submodule "src/libs/isotp-c"]
-	path = src/libs/isotp-c
-	url = https://github.com/openxc/isotp-c
 [submodule "src/libs/uds-c"]
 	path = src/libs/uds-c
 	url = https://github.com/openxc/uds-c

.travis.yml

@@ -1,12 +1,19 @@
 language: c
-dist: xenial
+dist: Bionic
 compiler:
 - clang
 script: PLATFORM=TESTING make test
 install:
 - gem install coveralls-lcov
 before_install:
 - sudo apt-get update -qq
+- sudo add-apt-repository ppa:deadsnakes/ppa -y
+- sudo apt-get update -qq
+- sudo apt-get install python3.6 -y
+- sudo rm /usr/bin/python3
+- sudo ln -s /usr/bin/python3.6 /usr/bin/python3
+- wget https://bootstrap.pypa.io/get-pip.py
+- sudo python3 get-pip.py
 - if [ `uname -m` = x86_64 ]; then sudo apt-get install -qq -y libsubunit-dev; fi
 - travis_wait script/bootstrap.sh
 - cd src

CHANGELOG.mkd

@@ -1,5 +1,20 @@
 # OpenXC Vehicle Interface Firmware Changelog
 
+## v8.0.0
+* BREAKING: This version requires updates to the virtual machine. 'vagrant up --provision' must be run after 'git pull'.
+* BREAKING: VI-Firmware 8.0.0 is not backwards compatable due to memory management updates and must be used with OpenXC-Python 2.0.0 or greater.
+* Update: Update Travis to Bionic.
+* Update: Update Vagrant to Bionic.
+* Update: Update Vagrant to dynamically allocate vcpus based on host cpu count.
+* Update: Update vm to use python3.
+* Fix: Out of memory error, drastically reduced memory cost per message.
+* Fix: Fix gitdir for submodules allowing host machine to still use git on vi-firmware.
+* Fix: Cleaned up multiple imports of the isotp and uds-c.
+* Fix: Fix bug with ignition detection.
+* Feature: Added code to remove signals.cpp when compiling emulator firmware.
+* Feature: Added checks to vm for old python and vm versions.
+* Feature: Added the ability for the user to set the pin on the bluetooth module.
+
 ## v7.3.0
 
 * BREAKING: Removed fine_odometer_since_restart from emulator.
@@ -21,8 +36,6 @@
 	- Updated flash instructions
 * Feature: Add PLATFORM command
 
-
-
 ## v7.2.0
 
 * BREAKING: This version requires updates to bootstrap. 'vagrant up --provision' must be run after 'git pull'.

README.rst

@@ -4,7 +4,7 @@ OpenXC Vehicle Interface Firmware
 
 .. image:: /docs/_static/logo.png
 
-:Version: 7.3.0
+:Version: 8.0.0
 :Web: http://openxcplatform.com
 :Documentation: http://vi-firmware.openxcplatform.com
 :Source: http://github.com/openxc/vi-firmware

Vagrantfile

@@ -5,12 +5,25 @@
 # Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
 VAGRANTFILE_API_VERSION = "2"
 
+def total_cpus
+  require 'etc'
+  Etc.nprocessors
+end
+
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # For a complete reference, please see the online documentation at
   # vagrantup.com.
 
   # Every Vagrant virtual environment requires a box to build off of.
-  config.vm.box = "ubuntu/xenial64"
+  config.vm.box = "bento/ubuntu-18.04"
+  config.vm.box_version = "201912.03.0"
+  config.vm.provider :virtualbox do |v|
+    if total_cpus > 1
+      v.cpus = total_cpus - 1
+	else
+	  v.cpus = 1
+	end
+  end
 
   # Check for proxy enviroment variable and set it
   if ENV['HTTP_PROXY'] || ENV['HTTPS_PROXY']
@@ -31,7 +44,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     end
   end
 
-
+  config.vm.box_download_insecure = true
   config.vm.provision "shell", privileged: false, keep_color: true do |s|
     s.inline = "ln -fs /vagrant vi-firmware;"
     s.inline += "VAGRANT=1 vi-firmware/script/bootstrap.sh"

docs/conf.py

@@ -49,9 +49,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '7.3.0'
+version = '8.0.0'
 # The full version, including alpha/beta/rc tags.
-release = '7.3.0'
+release = '8.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/config/code-examples.rst

@@ -64,8 +64,9 @@ In ``my_handlers.cpp``:
 .. code-block:: cpp
 
    /* Round the value down to 0 if it's less than 100. */
-   float ourRoundingDecoder(CanSignal* signal, CanSignal* signals,
-         int signalCount, float value, bool* send) {
+   float ourRoundingDecoder(const CanSignal* signal, const CanSignal* signals,
+         SignalManager* signalManager, SignalManager* signalManagers,
+		 int signalCount, Pipeline* pipeline, float value, bool* send) {
       if(value < 100) {
          value = 0;
       }
@@ -141,9 +142,10 @@ we look up the sign signal and use its value to transform the absolute value. In
 
    /* Load the last value for the sign signal and multiply the absolute value
    by it. */
-   float ourRoundingDecoder(CanSignal* signal, CanSignal* signals,
-         int signalCount, float value, bool* send) {
-       CanSignal* signSignal = lookupSignal("sign_of_signal",
+   float ourRoundingDecoder(const CanSignal* signal, const CanSignal* signals,
+         SignalManager* signalManager, SignalManager* signalManagers,
+		 int signalCount, Pipeline* pipeline, float value, bool* send) {
+       SignalManager* signSignal = lookupSignalManagerDetails("sign_of_signal",
                signals, signalCount);
 
        if(signSignal == NULL) {
@@ -271,8 +273,9 @@ In ``my_handlers.cpp``:
      *
      * This type signature is required for all custom message handlers.
      */
-    void latitudeMessageHandler(CanMessage* message, CanSignal* signals,
-            int signalCount, Pipeline* pipeline) {
+    void latitudeMessageHandler(const CanSignal* signal, const CanSignal* signals,
+	SignalManager* signalManager, SignalManager* signalManagers,
+    int signalCount, Pipeline* pipeline, float value, bool* send) {
         // Retrieve the CanSignal struct representations of the 3 latitude
         // component signals. These are still included in the firmware build
         // when the 'ignore' flag was true for the signals.
@@ -482,8 +485,9 @@ In ``my_handlers.cpp``:
 .. code-block:: cpp
 
    /* Ignore the signal if the value is less than 100 */
-   float ourFilteringDecoder(CanSignal* signal, CanSignal* signals,
-         int signalCount, float value, bool* send) {
+   float ourFilteringDecoder(const CanSignal* signal, const CanSignal* signals,
+         SignalManager* signalManager, SignalManager* signalManagers,
+		 int signalCount, Pipeline* pipeline, float value, bool* send) {
       if(value < 100) {
          *send = false;
       }

docs/index.rst

@@ -4,7 +4,7 @@ OpenXC Vehicle Interface Firmware
 
 .. image:: /_static/logo.png
 
-:Version: 7.3.0
+:Version: 8.0.0
 :Web: http://openxcplatform.com
 :Documentation: http://vi-firmware.openxcplatform.com
 :Source: http://github.com/openxc/vi-firmware

examples/diagnostic_handlers.cpp

@@ -1,5 +1,5 @@
 #include "can/canread.h"
 
-float handleMyDiagRequest(const DiagnosticResponse* response, float parsedPayload) {
-    return response->payload[0] * 3 + response->payload[1] * 2;
+void handleMyDiagRequest(const DiagnosticResponse* response, float parsedPayload, char* str_buf, int buf_size) {
+    snprintf(str_buf, buf_size, "%f", (float) (response->payload[0] * 3 + response->payload[1] * 2));
 }

examples/handlers.cpp

@@ -2,12 +2,14 @@
 
 using openxc::can::read::publishNumericalMessage;
 
-void handleSteeringWheelMessage(CanMessage* message,
-        CanSignal* signals, int signalCount, Pipeline* pipeline) {
+void handleSteeringWheelMessage(const CanSignal* signal, const CanSignal* signals, 
+	SignalManager* signalManager, SignalManager* signalManagers, int signalCount, 
+	CanMessage* message, Pipeline* pipeline) {
     publishNumericalMessage("latitude", 42.0, pipeline);
 }
 
-openxc_DynamicField handleInverted(CanSignal* signal, CanSignal* signals,
+openxc_DynamicField handleInverted(const CanSignal* signal, const CanSignal* signals,
+	SignalManager* signalManager, SignalManager* signalManagers,
         int signalCount, float value, bool* send) {
     return openxc::payload::wrapNumber(value * -1);
 }

fabfile.py

@@ -1,10 +1,11 @@
 import os
+import subprocess
 
 from fabric.api import *
 from fabric.colors import green, yellow, red
 from fabric.contrib.console import confirm
 
-from prettyprint import pp
+import pprint
 import copy
 import re
 
@@ -93,6 +94,7 @@ def make_tag():
     if confirm(yellow("Tag this release?"), default=True):
         print(green("The last 5 tags were: "))
         tags = local('git tag | tail -n 20', capture=True)
+        pp = pprint.PrettyPrinter()
         pp(sorted(tags.split('\n'), compare_versions, reverse=True))
         prompt("New release tag in the format vX.Y[.Z]?", 'tag',
                 validate=VERSION_PATTERN)
@@ -161,19 +163,19 @@ def build_options():
         options['DEFAULT_POWER_MANAGEMENT'] = "OBD2_IGNITION_CHECK"
         options['ENVIRONMENT_MODE'] = env.mode
     return " ".join((build_option(key, value)
-        for key, value in options.iteritems()))
+        for key, value in options.items()))
 
 def compile_firmware(build_name, target_path):
     with lcd("src"):
-        for board_name, board in env.boards.iteritems():
+        for board_name, board in env.boards.items():
             env.board = board_name
             build(capture=True, do_clean=True)
             local("cp build/%s/vi-firmware-%s.%s %s/vi-%s-firmware-%s-ct%s.%s"
                     % (board['name'], board['name'], board['extension'],
                         target_path, build_name, board['name'],
                         env.firmware_release, board['extension']))
 
-        for board_name, board in env.msd_boards.iteritems():
+        for board_name, board in env.msd_boards.items():
             msd_enable()
             env.board = board_name
             build(capture=True, do_clean=True)
@@ -187,6 +189,10 @@ def compress_release(source, archive_path):
     with lcd(os.path.dirname(source)):
         local("zip -r %s %s" % (archive_path, os.path.basename(source)))
 
+def quiet_build():
+    output = subprocess.call("make", cwd="src", env={"PLATFORM": "FORDBOARD"}, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
+    return output
+
 @task
 def emulator():
     env.mode = 'emulator'
@@ -226,10 +232,29 @@ def test(long=False):
                 "a released version in master branch"))
 
     with(lcd("src")):
+        if os.path.exists("src/signals.cpp"):
+            local("mv signals.cpp signals.cpp.oxc")
+
+        local("touch signals.cpp")
+        failed_build = quiet_build()
+        local("cp samples/valid_signals.cpp signals.cpp")
+        passed_build = quiet_build()
+
+        tests_passed = 0
+        if failed_build == 2:
+            tests_passed = tests_passed + 1
+        if passed_build == 0:
+            tests_passed = tests_passed + 1
+
+        print(f"Make tests passed: {tests_passed}/2")
+
+        if os.path.exists("src/signals.cpp.oxc"):
+            local("mv signals.cpp.oxc signals.cpp")
+
         if long in (True, 'True', 'true'):
-            local("PLATFORM=TESTING make -j4 test_long")
+            local("PLATFORM=TESTING make -j1 test_long")
         else:
-            local("PLATFORM=TESTING make -j4 test")
+            local("PLATFORM=TESTING make -j1 test")
 
 @task
 def functional_test_flash(skip_flashing=False):
@@ -336,8 +361,15 @@ def build(capture=False, do_clean=False):
     options = build_options()
     with lcd("%s/src" % env.root_dir):
         if do_clean:
-            clean();
-        output = local("%s make -j4" % options, capture=capture)
+            clean()
+        if env.mode == 'emulator' and os.path.exists("src/signals.cpp"):
+            with quiet():
+                local(f"mv signals.cpp signals.cpp.oxc")
+        output = local("%s make -j1 " % options, capture=capture)
+        if env.mode == 'emulator' and os.path.exists("src/signals.cpp.oxc"):
+            with quiet():
+                local(f"mv signals.cpp.oxc signals.cpp")
+            print("Since this is an emulator build, your \'signals.cpp\' was backed up and restored.")
         if output.failed:
             puts(output)
             abort(red("Building %s failed" % board_options['name']))

script/bootstrap/ci-requirements.txt

@@ -1 +1 @@
-openxc==0.13.0
+openxc==2.0.0

script/bootstrap/common.sh

@@ -97,8 +97,8 @@ if [ -z $COMMON_SOURCED ]; then
                 if [ $DISTRO == "arch" ]; then
                     $SUDO_CMD pacman -S $1
                 elif [ $DISTRO == "Ubuntu" ]; then
-                    $SUDO_CMD apt-get update -qq
-                    $SUDO_CMD apt-get install $1 -y
+                    $SUDO_CMD apt-get update -qq -y
+                    $SUDO_CMD apt install -y $1 
                 else
                     echo
                     echo "Missing $1 - install it using your distro's package manager or build from source"
@@ -207,22 +207,33 @@ pre-configured Vagrant environment. See the docs for more information."
     fi
     set -e
 
-    #https://cryptography.io/en/latest/installation/#building-cryptography-on-linux
-    #cryptography is dependency of pyparsing - need to ensure other packages first
-    #otherwise cffi error
-    if [ $OS == "linux" ]; then
-	echo "Installing libffi-dev"
-	_install "libffi-dev"
-    fi
+    echo "Fixing gitdir for nested submodules"
+
+    # https://marc.info/?l=git&m=145937129606918&w=2
+    # There's a bug with the git submodule update command (still unfixed as of v2.23.0) that causes nested submodules to use an absolue
+    # path for their gitdir, whereas everything else uses relative paths. Since the project root in this context
+    # is /vagrant/, the gitdirs for the nested submodules will be incorrect and consequentially most git commands won't 
+    # work (unless you're in a vagrant machine). For now we'll just rewrite the gitdirs for the nested submodules to use relative paths
+    sed -i -e 's/\/vagrant\//..\/..\/..\/..\/..\//g' src/libs/AT-commander/lpc17xx/BSP/.git
+    sed -i -e 's/\/vagrant\//..\/..\/..\/..\/..\//g' src/libs/AT-commander/lpc17xx/CDL/.git
+    sed -i -e 's/\/vagrant\//..\/..\/..\/..\/..\//g' src/libs/AT-commander/lpc17xx/emqueue/.git
+    sed -i -e 's/\/vagrant\//..\/..\/..\/..\/..\//g' src/libs/openxc-message-format/libs/nanopb/.git
+    sed -i -e 's/\/vagrant\//..\/..\/..\/..\/..\//g' src/libs/uds-c/deps/isotp-c/.git
+    sed -i -e 's/\/vagrant\//..\/..\/..\/..\/..\/..\/..\//g' src/libs/uds-c/deps/isotp-c/deps/bitfield-c/.git
+
+    # Force git to use CR/LF line endings. The default from within a vagrant machine is different, so without this
+    # it'll think all of the files have changed when inside a vagrant machine
+    git config --global core.autocrlf true
 
-    if ! command -v python >/dev/null 2>&1; then
+    if ! command -v python3 >/dev/null 2>&1; then
         echo "Installing Python..."
-        _install "python"
+        _install "python3"
+        $SUDO_CMD ln -s /usr/bin/python3 /usr/bin/python
     fi
 
-    if ! command -v pip >/dev/null 2>&1; then
-		curl -Ss https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
-        $SUDO_CMD python /tmp/get-pip.py
+    if ! command -v pip3 >/dev/null 2>&1; then
+        _install python3-pip
+        $SUDO_CMD ln -s /usr/bin/pip3 /usr/bin/pip
     fi
 
     PIP_SUDO_CMD=
@@ -231,9 +242,9 @@ pre-configured Vagrant environment. See the docs for more information."
         PIP_SUDO_CMD=$SUDO_CMD
     fi
 
-    $PIP_SUDO_CMD pip install --src dependencies --pre -r $BOOTSTRAP_DIR/ci-requirements.txt
+    $PIP_SUDO_CMD pip3 install --src dependencies --pre -r $BOOTSTRAP_DIR/ci-requirements.txt
     if [ -z $CI ]; then
-        $PIP_SUDO_CMD pip install --src dependencies --pre -r $BOOTSTRAP_DIR/pip-requirements.txt
+        $PIP_SUDO_CMD pip3 install --src dependencies --pre -r $BOOTSTRAP_DIR/pip-requirements.txt
     fi
 
     if ! command -v clang >/dev/null 2>&1; then

script/bootstrap/pip-requirements.txt

@@ -1,4 +1,5 @@
-Fabric==1.13.1
-ecdsa==0.13.3
+Fabric3==1.14.post1
+ecdsa==0.13
 prettyprint==0.1.5
 pyparsing==2.2.0
+openxc==2.0.0