Generate plugin_firmware_index.json with new boards data (#172)

* Add arduino uno r4 to boards.json

* Add arduino uno r4 to the generated boards json

* Do not use precompiled sketches and skip core installation for new boards

* Hard code uploader_plugin and additional_tools

* Add ESP32-S3 firmwares

* replace uno r4 placeholder fw with released ones

* Add mkr wifi 1010 to old_boards

* Create a different json for the new boards

* Use the same directory for both of the json files

* Add plugin firmware index generation steps to the workflow

* Reduce complexity using a new function to create the starting dictionary

---------

Co-authored-by: Umberto Baldi <u.baldi@arduino.cc>

.github/workflows/generate-index.yml

@@ -56,9 +56,12 @@ jobs:
           cd $GITHUB_WORKSPACE
           task poetry:install-deps
 
-      - name: Generate index
+      - name: Generate plugin firmware index
         run: poetry run ./generator.py -a $(which arduino-cli)
 
+      - name: Generate module firmware index
+        run: poetry run ./generator.py -a $(which arduino-cli) --no-new
+
         # fix `gpg: signing failed: Inappropriate ioctl for device`
         # https://github.com/keybase/keybase-issues/issues/2798
       - name: Import GPG key
@@ -69,16 +72,24 @@ jobs:
 
         # disable gpg pass prompt
         # https://stackoverflow.com/questions/49072403/suppress-the-passphrase-prompt-in-gpg-command
-      - name: sign the json
+      - name: sign the module firmware index json
         run: |
           gpg \
             --pinentry-mode=loopback \
             --passphrase "${{ secrets.PASSPHRASE }}" \
             --output boards/module_firmware_index.json.sig \
             --detach-sign boards/module_firmware_index.json
 
+      - name: sign the plugin firmware index json
+        run: |
+          gpg \
+            --pinentry-mode=loopback \
+            --passphrase "${{ secrets.PASSPHRASE }}" \
+            --output boards/plugin_firmware_index.json.sig \
+            --detach-sign boards/plugin_firmware_index.json
+
       - name: create the gzip
-        run: gzip --keep boards/module_firmware_index.json
+        run: gzip --keep boards/module_firmware_index.json boards/plugin_firmware_index.json
 
       - name: s3 sync
         run: |

generator/generator.py

@@ -26,6 +26,42 @@
 DOWNLOAD_URL = "https://downloads.arduino.cc/arduino-fwuploader"
 
 
+# create a different dictionary for new boards
+def create_boards_dictionary(new):
+    boards = {
+        "arduino:samd:mkr1000": {"fqbn": "arduino:samd:mkr1000", "firmware": []},
+        "arduino:samd:mkrwifi1010": {
+            "fqbn": "arduino:samd:mkrwifi1010",
+            "firmware": [],
+        },
+        "arduino:samd:nano_33_iot": {
+            "fqbn": "arduino:samd:nano_33_iot",
+            "firmware": [],
+        },
+        "arduino:samd:mkrvidor4000": {
+            "fqbn": "arduino:samd:mkrvidor4000",
+            "firmware": [],
+        },
+        "arduino:megaavr:uno2018": {"fqbn": "arduino:megaavr:uno2018", "firmware": []},
+        "arduino:mbed_nano:nanorp2040connect": {
+            "fqbn": "arduino:mbed_nano:nanorp2040connect",
+            "firmware": [],
+        },
+    }
+    if new:
+        boards = {
+            "arduino:renesas_uno:unor4wifi": {
+                "fqbn": "arduino:renesas_uno:unor4wifi",
+                "firmware": [],
+                # "uploader_plugin" and "additional_tools" need to be hard coded because
+                # there is no way to retrieve them dinamically
+                "uploader_plugin": "arduino:uno-r4-wifi-fwuploader@1.0.0",
+                "additional_tools": ["arduino:espflash@2.0.0", "arduino:bossac@1.9.1-arduino5"],
+            },
+        }
+    return boards
+
+
 # handle firmware name
 def get_firmware_file(module, simple_fqbn, version):
     firmware_full_path = Path(__file__).parent.parent / "firmwares" / module / version
@@ -222,35 +258,26 @@ def create_upload_data(fqbn, installed_cores):  # noqa: C901
     return upload_data
 
 
-def generate_boards_json(input_data, arduino_cli_path):
-    boards = {
-        "arduino:samd:mkr1000": {"fqbn": "arduino:samd:mkr1000", "firmware": []},
-        "arduino:samd:mkrwifi1010": {
-            "fqbn": "arduino:samd:mkrwifi1010",
-            "firmware": [],
-        },
-        "arduino:samd:nano_33_iot": {
-            "fqbn": "arduino:samd:nano_33_iot",
-            "firmware": [],
-        },
-        "arduino:samd:mkrvidor4000": {
-            "fqbn": "arduino:samd:mkrvidor4000",
-            "firmware": [],
-        },
-        "arduino:megaavr:uno2018": {"fqbn": "arduino:megaavr:uno2018", "firmware": []},
-        "arduino:mbed_nano:nanorp2040connect": {
-            "fqbn": "arduino:mbed_nano:nanorp2040connect",
-            "firmware": [],
-        },
-    }
+def generate_boards_json(input_data, arduino_cli_path, new_boards):
+    # List of old boards that need precompiled sketch data and uploader information obtained through platform.txt.
+    old_boards = [
+        "arduino:samd:mkr1000",
+        "arduino:samd:mkrwifi1010",
+        "arduino:samd:nano_33_iot",
+        "arduino:samd:mkrvidor4000",
+        "arduino:megaavr:uno2018",
+        "arduino:mbed_nano:nanorp2040connect",
+    ]
+
+    boards = create_boards_dictionary(new_boards)
 
     # Gets the installed cores
     res = arduino_cli(cli_path=arduino_cli_path, args=["core", "list", "--format", "json"])
     installed_cores = {c["id"]: c for c in json.loads(res)}
 
     # Verify all necessary cores are installed
     # TODO: Should we check that the latest version is installed too?
-    for fqbn in boards.keys():
+    for fqbn in old_boards:
         core_id = ":".join(fqbn.split(":")[:2])
         if core_id not in installed_cores:
             print(f"Board {fqbn} is not installed, install its core {core_id}")
@@ -259,8 +286,9 @@ def generate_boards_json(input_data, arduino_cli_path):
     for fqbn, data in input_data.items():
         simple_fqbn = fqbn.replace(":", ".")
 
-        boards[fqbn]["loader_sketch"] = create_precomp_sketch_data(simple_fqbn, "loader")
-        boards[fqbn]["version_sketch"] = create_precomp_sketch_data(simple_fqbn, "getversion")
+        if fqbn in old_boards:
+            boards[fqbn]["loader_sketch"] = create_precomp_sketch_data(simple_fqbn, "loader")
+            boards[fqbn]["version_sketch"] = create_precomp_sketch_data(simple_fqbn, "getversion")
 
         for firmware_version in data["versions"]:
             module = data["moduleName"]
@@ -278,7 +306,8 @@ def generate_boards_json(input_data, arduino_cli_path):
                 boards[fqbn]["name"] = board["name"]
                 break
 
-        boards[fqbn].update(create_upload_data(fqbn, installed_cores))
+        if fqbn in old_boards:
+            boards[fqbn].update(create_upload_data(fqbn, installed_cores))
 
     boards_json = []
     for _, b in boards.items():
@@ -296,18 +325,31 @@ def generate_boards_json(input_data, arduino_cli_path):
         help="Path to arduino-cli executable",
         required=True,
     )
+    parser.add_argument(
+        "--new",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Generate the index for old boards",
+    )
     args = parser.parse_args(sys.argv[1:])
 
+    if args.new:
+        input_file = "new_boards.json"
+        output_file = "plugin_firmware_index.json"
+    else:
+        input_file = "boards.json"
+        output_file = "module_firmware_index.json"
+
     # raw_boards.json has been generated using --get_available_for FirmwareUploader (version 0.1.8) flag.
     # It has been edited a bit to better handle parsing.
-    with open("boards.json", "r") as f:
+    with open(input_file, "r") as f:
         boards = json.load(f)
 
-    boards_json = generate_boards_json(boards, args.arduino_cli)
+    boards_json = generate_boards_json(boards, args.arduino_cli, args.new)
 
-    Path("boards").mkdir()
+    Path("boards").mkdir(exist_ok=True)
 
-    with open("boards/module_firmware_index.json", "w") as f:
+    with open("boards/" + output_file, "w") as f:
         json.dump(boards_json, f, indent=2)
 
 # board_index.json must be formatted like so:

generator/new_boards.json

@@ -0,0 +1,6 @@
+{
+  "arduino:renesas_uno:unor4wifi": {
+    "moduleName": "ESP32-S3",
+    "versions": ["0.1.0", "0.2.0"]
+  }
+}