Merge pull request #68 from Xilinx/fix/linting

Bring linting inline with FINN repo

.github/workflows/pre-commit.yml

@@ -0,0 +1,24 @@
+name: Pre-commit
+
+on:
+  pull_request:
+    branches: [ main, dev ]
+  push:
+    branches: [ main, dev ]
+
+jobs:
+  lint:
+    name: Lint PR or Push to DEV
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Run Lint
+        uses: pre-commit/[email protected]

.pre-commit-config.yaml

@@ -1,38 +1,71 @@
+# Copyright (C) 2023, Advanced Micro Devices, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of FINN nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+exclude: '^docs/conf.py'
+
 default_language_version:
-    python: python3
+    python: python3.10
 
 repos:
-- repo: git://github.com/pre-commit/pre-commit-hooks
-  rev: v3.2.0
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
   hooks:
   - id: trailing-whitespace
+    exclude: '\.dat$'
   - id: check-added-large-files
   - id: check-ast
   - id: check-json
   - id: check-merge-conflict
   - id: check-xml
   - id: check-yaml
   - id: debug-statements
+    exclude: '^src/finn/builder/build_dataflow.py$'
   - id: end-of-file-fixer
   - id: requirements-txt-fixer
   - id: mixed-line-ending
     args: ['--fix=no']
 
-- repo: git://github.com/PyCQA/isort
-  rev: 5.5.3
+- repo: https://github.com/PyCQA/isort
+  rev: 5.12.0
   hooks:
   - id: isort
 
-- repo: git://github.com/psf/black
-  rev: stable
+- repo: https://github.com/psf/black
+  rev: 23.3.0
   hooks:
   - id: black
     language_version: python3
+    args: [--line-length=100]
 
-- repo: https://gitlab.com/pycqa/flake8
-  rev: 3.8.3
+- repo: https://github.com/PyCQA/flake8
+  rev: 6.0.0
   hooks:
   - id: flake8
     # black-compatible flake-8 config
-    args: ['--max-line-length=88',  # black default
+    args: ['--max-line-length=100',  # black default
            '--extend-ignore=E203']  # E203 is not PEP8 compliant

AMD-license-agreement-for-non-commercial-models.md

@@ -1,14 +1,14 @@
-ADVANCED MICRO DEVICES, INC. 
-LICENSE AGREEMENT FOR NON-COMMERCIAL MODELS 
+ADVANCED MICRO DEVICES, INC.
+LICENSE AGREEMENT FOR NON-COMMERCIAL MODELS
 
-Trained Models: 
+Trained Models:
 
 bincop-cnv, cnv-w1a1, cnv-w1a2, cnv-w2a2, kwsmlp-w3a3, mobilenetv1-w4a4, resnet50-w1a2, tfc-w1a1, tfc-w1a2, tfc-w2a2, unsw_nb15-mlp-w2a2, vgg10-radioml-w4a4
 
-This License Agreement for Non-Commercial Models (“Agreement”) is a legal agreement between you (either an individual or an entity) and Advanced Micro Devices, Inc. on behalf of itself and its subsidiaries and affiliates (collectively “AMD”). DO NOT USE THE TRAINED MODELS IDENTIFIED ABOVE UNTIL YOU HAVE CAREFULLY READ THIS AGREEMENT. BY USING, INSTALLING, MODIFYING, COPYING, TRAINING, BENCHMARKING, OR DISTRIBUTING THE TRAINED MODELS, YOU AGREE TO AND ACCEPT ALL TERMS AND CONDITIONS OF THIS AGREEMENT. If you do not accept these terms, do not use the Trained Models. 
+This License Agreement for Non-Commercial Models (“Agreement”) is a legal agreement between you (either an individual or an entity) and Advanced Micro Devices, Inc. on behalf of itself and its subsidiaries and affiliates (collectively “AMD”). DO NOT USE THE TRAINED MODELS IDENTIFIED ABOVE UNTIL YOU HAVE CAREFULLY READ THIS AGREEMENT. BY USING, INSTALLING, MODIFYING, COPYING, TRAINING, BENCHMARKING, OR DISTRIBUTING THE TRAINED MODELS, YOU AGREE TO AND ACCEPT ALL TERMS AND CONDITIONS OF THIS AGREEMENT. If you do not accept these terms, do not use the Trained Models.
 
-1. Subject to your compliance with this Agreement, AMD grants you a license to use, modify, and distribute the Trained Models solely for non-commercial and research purposes. This means you may use the Trained Models for benchmarking, testing, and evaluating the Trained Models (including non-commercial research undertaken by or funded by a commercial entity) but you cannot use the Trained Models in any commercial offering, including as part of a product or service you use or provide to others in exchange for money or other consideration. 
+1. Subject to your compliance with this Agreement, AMD grants you a license to use, modify, and distribute the Trained Models solely for non-commercial and research purposes. This means you may use the Trained Models for benchmarking, testing, and evaluating the Trained Models (including non-commercial research undertaken by or funded by a commercial entity) but you cannot use the Trained Models in any commercial offering, including as part of a product or service you use or provide to others in exchange for money or other consideration.
 
-2. Your license to the Trained Models is subject to the following conditions: (a) you cannot alter any copyright, trademark, or other notice in the Trained Models; (b) you cannot sublicense or distribute the Trained Models under any other terms or conditions; (c) you cannot use AMD’s trademarks in your applications or technologies in a way that suggests your applications or technologies are endorsed by AMD; (d) if you distribute a Trained Model, you must provide corresponding source code for such Trained Model; and (e) if the Trained Models include any code or content subject to an open source license or third party license (“Third Party Materials”), you agree to comply with such license terms. 
+2. Your license to the Trained Models is subject to the following conditions: (a) you cannot alter any copyright, trademark, or other notice in the Trained Models; (b) you cannot sublicense or distribute the Trained Models under any other terms or conditions; (c) you cannot use AMD’s trademarks in your applications or technologies in a way that suggests your applications or technologies are endorsed by AMD; (d) if you distribute a Trained Model, you must provide corresponding source code for such Trained Model; and (e) if the Trained Models include any code or content subject to an open source license or third party license (“Third Party Materials”), you agree to comply with such license terms.
 
 3. THE TRAINED MODELS (INCLUDING THIRD PARTY MATERIALS, IF ANY) ARE PROVIDED “AS IS” AND WITHOUT A WARRANTY OF ANY KIND, WHETHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. YOU BEAR ALL RISK OF USING THE TRAINED MODELS (INCLUDING THIRD PARTY MATERIALS, IF ANY) AND YOU AGREE TO RELEASE AMD FROM ANY LIABILITY OR DAMAGES FOR ANY CLAIM OR ACTION ARISING OUT OF OR IN CONNECTION WITH YOUR USE OF THE TRAINED MODELS AND/OR THIRD PARTY MATERIALS

build/cybersecurity-mlp/README.md

@@ -19,4 +19,4 @@ cd $FINN_EXAMPLES/build/finn
 3. The generated outputs will be under `cybersecurity-mlp/output_<topology>_<platform>`. You can find a description of the generated files [here](https://finn-dev.readthedocs.io/en/latest/command_line.html#simple-dataflow-build-mode).
 
 # Where did the ONNX model file come from?
-The ONNX model is created and exported prior to the build flow is launched. You can find the details of this process in the `cybersecurity-mlp/custom_steps.py` file.
+The ONNX model is created and exported prior to the build flow is launched. You can find the details of this process in the `cybersecurity-mlp/custom_steps.py` file.

build/cybersecurity-mlp/build.py

@@ -40,6 +40,7 @@
 # Note: only zynq platforms currently tested
 platforms_to_build = zynq_platforms + alveo_platforms
 
+
 # determine which shell flow to use for a given platform
 def platform_to_shell(platform):
     if platform in zynq_platforms:
@@ -49,6 +50,7 @@ def platform_to_shell(platform):
     else:
         raise Exception("Unknown platform, can't determine ShellFlowType")
 
+
 # Define model name
 model_name = "unsw_nb15-mlp-w2a2"
 
@@ -71,21 +73,21 @@ def platform_to_shell(platform):
     os.makedirs(platform_dir, exist_ok=True)
     # Set up the build configuration for this model
     cfg = build_cfg.DataflowBuildConfig(
-        output_dir = "output_%s_%s" % (model_name, release_platform_name),
-        mvau_wwidth_max = 80,
-        target_fps = 1000000,
-        synth_clk_period_ns = 10.0,
-        board = platform_name,
-        shell_flow_type = shell_flow_type,
-        vitis_platform = vitis_platform,
-        vitis_opt_strategy = build_cfg.VitisOptStrategyCfg.PERFORMANCE_BEST,
-        generate_outputs = [
+        output_dir="output_%s_%s" % (model_name, release_platform_name),
+        mvau_wwidth_max=80,
+        target_fps=1000000,
+        synth_clk_period_ns=10.0,
+        board=platform_name,
+        shell_flow_type=shell_flow_type,
+        vitis_platform=vitis_platform,
+        vitis_opt_strategy=build_cfg.VitisOptStrategyCfg.PERFORMANCE_BEST,
+        generate_outputs=[
             build_cfg.DataflowOutputType.PYNQ_DRIVER,
             build_cfg.DataflowOutputType.ESTIMATE_REPORTS,
             build_cfg.DataflowOutputType.BITFILE,
             build_cfg.DataflowOutputType.DEPLOYMENT_PACKAGE,
         ],
-        save_intermediate_models=True
+        save_intermediate_models=True,
     )
 
     # Export MLP model to FINN-ONNX
@@ -94,11 +96,7 @@ def platform_to_shell(platform):
     build.build_dataflow_cfg(model, cfg)
     # Copy bitfiles into release dir if found
     bitfile_gen_dir = cfg.output_dir + "/bitfile"
-    filtes_to_check_and_copy = [
-        "finn-accel.bit",
-        "finn-accel.hwh",
-        "finn-accel.xclbin"
-    ]
+    filtes_to_check_and_copy = ["finn-accel.bit", "finn-accel.hwh", "finn-accel.xclbin"]
     for f in filtes_to_check_and_copy:
         src_file = bitfile_gen_dir + "/" + f
         dst_file = platform_dir + "/" + f.replace("finn-accel", model_name)

build/cybersecurity-mlp/custom_steps.py

@@ -36,17 +36,14 @@
 import brevitas.onnx as bo
 from brevitas.quant_tensor import QuantTensor
 
+
 # Define export wrapper
 class CybSecMLPForExport(nn.Module):
     def __init__(self, my_pretrained_model):
         super(CybSecMLPForExport, self).__init__()
         self.pretrained = my_pretrained_model
         self.qnt_output = QuantIdentity(
-            quant_type='binary',
-            scaling_impl_type='const',
-            bit_width=1,
-            min_val=-1.0,
-            max_val=1.0
+            quant_type="binary", scaling_impl_type="const", bit_width=1, min_val=-1.0, max_val=1.0
         )
 
     def forward(self, x):
@@ -55,9 +52,10 @@ def forward(self, x):
         # input range for the trained network
         x = (x + torch.tensor([1.0]).to("cpu")) / 2.0
         out_original = self.pretrained(x)
-        out_final = self.qnt_output(out_original) # output as {-1, 1}
+        out_final = self.qnt_output(out_original)  # output as {-1, 1}
         return out_final
 
+
 def custom_step_mlp_export(model_name):
     # Define model parameters
     input_size = 593
@@ -82,12 +80,12 @@ def custom_step_mlp_export(model_name):
         nn.BatchNorm1d(hidden3),
         nn.Dropout(0.5),
         QuantReLU(bit_width=act_bit_width),
-        QuantLinear(hidden3, num_classes, bias=True, weight_bit_width=weight_bit_width)
+        QuantLinear(hidden3, num_classes, bias=True, weight_bit_width=weight_bit_width),
     )
 
     # Load pre-trained weights
     assets_dir = pk.resource_filename("finn.qnn-data", "cybsec-mlp/")
-    trained_state_dict = torch.load(assets_dir+"/state_dict.pth")["models_state_dict"][0]
+    trained_state_dict = torch.load(assets_dir + "/state_dict.pth")["models_state_dict"][0]
     model.load_state_dict(trained_state_dict, strict=False)
 
     # Network surgery: pad input size from 593 to 600 and convert bipolar to binary
@@ -112,10 +110,6 @@ def custom_step_mlp_export(model_name):
     )
 
     # Export to ONNX
-    bo.export_finn_onnx(
-        model_for_export, export_path=ready_model_filename, input_t=input_qt
-    )
+    bo.export_finn_onnx(model_for_export, export_path=ready_model_filename, input_t=input_qt)
 
     return ready_model_filename
-
-

build/kws/README.md

@@ -18,7 +18,7 @@ cd $FINN_EXAMPLES/build/finn
 bash run-docker.sh build_custom $FINN_EXAMPLES/build/kws
 ```
 
-3. The generated outputs will be under `kws/<timestamp>_output_<onnx_file_name>_<platform>`. 
+3. The generated outputs will be under `kws/<timestamp>_output_<onnx_file_name>_<platform>`.
 You can find a description of the generated files [here](https://finn-dev.readthedocs.io/en/latest/command_line.html#simple-dataflow-build-mode).
 The folder will additionally include the quantized inputs for verification (`all_validation_KWS_data_inputs_len_10102.npy`) and the expected outputs (`all_validation_KWS_data_outputs_len_10102.npy`).
-When running the network on hardware the validation should achieve an accuracy of 89.78 % with 9070 of the 10102 samples being classified correctly.  
+When running the network on hardware the validation should achieve an accuracy of 89.78 % with 9070 of the 10102 samples being classified correctly.

build/kws/build.py

@@ -67,13 +67,10 @@ def step_preprocess(model: ModelWrapper, cfg: DataflowBuildConfig):
     build_cfg.VerificationStepType.FOLDED_HLS_CPPSIM,
 ]
 
-model_name = (
-    "MLP_W3A3_python_speech_features_pre-processing_QONNX"
-)
+model_name = "MLP_W3A3_python_speech_features_pre-processing_QONNX"
 model_file = "models/" + model_name + ".onnx"
 
 # Change the ONNX opset from version 9 to 11, which adds support for the TopK node
-from qonnx.core.modelwrapper import ModelWrapper
 model = ModelWrapper(model_file)
 model.model.opset_import[0].version = 11
 model_file = model_file.replace(".onnx", "_opset-11.onnx")
@@ -83,12 +80,12 @@ def step_preprocess(model: ModelWrapper, cfg: DataflowBuildConfig):
 # create a release dir, used for finn-examples release packaging
 os.makedirs("release", exist_ok=True)
 platforms_to_build = ["Pynq-Z1"]
-last_output_dir=""
+last_output_dir = ""
 for platform_name in platforms_to_build:
     release_platform_name = platform_name
     platform_dir = "release/%s" % release_platform_name
     os.makedirs(platform_dir, exist_ok=True)
-    last_output_dir="output_%s_%s" % (model_name, release_platform_name)
+    last_output_dir = "output_%s_%s" % (model_name, release_platform_name)
     # Configure build
     cfg = build_cfg.DataflowBuildConfig(
         # steps=estimate_steps, generate_outputs=estimate_outputs,
@@ -121,7 +118,6 @@ def step_preprocess(model: ModelWrapper, cfg: DataflowBuildConfig):
             shutil.copy(src_file, dst_file)
 
 
-
 # Export quantized inputs
 print("Quantizing validation dataset.")
 parent_model = ModelWrapper(last_output_dir + "/intermediate_models/dataflow_parent.onnx")
@@ -157,8 +153,7 @@ def step_preprocess(model: ModelWrapper, cfg: DataflowBuildConfig):
         time_left = (len(data_arr) - (i + 1)) * time_per_sample
         time_left = datetime.timedelta(seconds=time_left)
         print(
-            f"Processed: {100*(i+1)/len(data_arr):.1f} [%], "
-            f"time left: {str(time_left)}",
+            f"Processed: {100*(i+1)/len(data_arr):.1f} [%], " f"time left: {str(time_left)}",
             end="\r",
         )
     print()
@@ -171,7 +166,5 @@ def step_preprocess(model: ModelWrapper, cfg: DataflowBuildConfig):
     # Save data
     export_path = f_name.replace(".npz", "_{}_len_{}.npy")
     print(f"Saving data to: {export_path}")
-    np.save(
-        export_path.format("inputs", len(pre_processed_inputs)), pre_processed_inputs
-    )
+    np.save(export_path.format("inputs", len(pre_processed_inputs)), pre_processed_inputs)
     np.save(export_path.format("outputs", len(label_arr)), label_arr)

build/mobilenet-v1/build.py

@@ -44,9 +44,9 @@
 model_name = "mobilenetv1-w4a4"
 
 # which platforms to build the networks for
-#zynq_platforms = ["ZCU102", "ZCU104"]
+# zynq_platforms = ["ZCU102", "ZCU104"]
 zynq_platforms = ["ZCU102"]
-#alveo_platforms = ["U50", "U200", "U250", "U280"]
+# alveo_platforms = ["U50", "U200", "U250", "U280"]
 alveo_platforms = ["U250"]
 platforms_to_build = zynq_platforms + alveo_platforms
 
@@ -165,4 +165,3 @@ def select_build_steps(platform):
         weight_files = os.listdir(weight_gen_dir)
         if weight_files:
             shutil.copytree(weight_gen_dir, weight_dst_dir)
-

build/mobilenet-v1/custom_steps.py

@@ -125,9 +125,7 @@ def step_mobilenet_slr_floorplan(model: ModelWrapper, cfg: DataflowBuildConfig):
     return model
 
 
-def step_mobilenet_convert_to_hls_layers_separate_th(
-    model: ModelWrapper, cfg: DataflowBuildConfig
-):
+def step_mobilenet_convert_to_hls_layers_separate_th(model: ModelWrapper, cfg: DataflowBuildConfig):
     mem_mode = cfg.default_mem_mode.value
     model = model.transform(to_hls.InferPool_Batch())
     model = model.transform(to_hls.InferConvInpGen())