Generate model with int4 weight ( maybe single op model )

[chunseoklee]

There are several options to generate this :

• circlechef : [onert] Let's support int4 weight #12741 (comment) (draft)
• https://www.tensorflow.org/model_optimization/guide/quantization/training_comprehensive_guide#use_custom_quantization_algorithm
• manual generation with flatc

[chunseoklee]

This is one op circle model( FullyConnected_INT4.zip ) generated by flatcing with json file below :

op: FC
I : f32 (1,2)
W: int4 (4,2) packed
A: f32 (1,4)

{
  operator_codes: [
    {                                                                                                                                                                                                                                                                                                                                                                                                                                      deprecated_builtin_code: 9,
      builtin_code: "FULLY_CONNECTED"
    }
  ],
  subgraphs: [
    {
      tensors: [
        {
          shape: [
            1,
            2
          ],
          buffer: 1,
          name: "in"
        },
        {
          shape: [
            4,
            2
          ],
          type: "INT4",
          buffer: 2,
          name: "weight",
          quantization: {
            min: [
              0.0
            ],
            max: [
              1.0
            ],
            scale: [
              0.000554
            ],
            zero_point: [
              0
            ]
          }
        },
        {
          shape: [
            4
          ],
          buffer: 3,
          name: "bias"
        },
        {
          shape: [
            1,
            4
          ],
          buffer: 4,
          name: "out"
        }
      ],
      inputs: [
        0,
      ],
      outputs: [
        3
      ],
      operators: [
        {
          inputs: [
            0,
            1,
            2
          ],
          outputs: [
            3
          ],
          builtin_options_type: "FullyConnectedOptions",
          builtin_options: {
          }
        }
      ],
      name: "main"
    }
  ],
  description: "ONE-tflite2circle",
  buffers: [
    {

    },
    {
    },
    { data : [16,0,255,0]
    },
    { data : [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
    },
    {
    },
  ]
}

[chunseoklee]

nnfw_api_test model : 367b9bb

ONE/tests/nnfw_api/src/GenModelTests/one_op_tests/FullyConnected.test.cc

Lines 60 to 87 in 367b9bb

	TEST_F(GenModelTest, OneOp_FullyConnected_Int4Weight)
	{
	CircleGen cgen;
	// clang-format off
	// This is packed weight for [0, 1, 0, 0, -1, -1 ,0, 0]
	std::vector<uint8_t> weight_data{ 16, 0, 255, 0 };
	std::vector<float> bias_data{ 0, 1, 2, 3 };
	// clang-format on
	uint32_t weight_buf = cgen.addBuffer(weight_data);
	uint32_t bias_buf = cgen.addBuffer(bias_data);
	int input = cgen.addTensor({{1, 2}, circle::TensorType::TensorType_FLOAT32});
	std::vector<float> weight_scales = {0.5};
	std::vector<int64_t> weight_zeropoints = {0};
	int weight = cgen.addTensor({{4, 2}, circle::TensorType::TensorType_INT4, weight_buf}, weight_scales, weight_zeropoints);
	int bias = cgen.addTensor({{4}, circle::TensorType::TensorType_FLOAT32, bias_buf});
	int output = cgen.addTensor({{1, 4}, circle::TensorType::TensorType_FLOAT32});
	cgen.addOperatorFullyConnected({{input, weight, bias}, {output}});
	cgen.setInputsAndOutputs({input}, {output});
	_context = std::make_unique<GenModelTestContext>(cgen.finish());
	_context->addTestCase(
	uniformTCD<float>({{1, 3}}, {{1.5 , 1, 0, 3}}));
	_context->setBackends({"cpu"});
	SUCCEED();
	}