diff --git a/compute/cker/include/cker/Types.h b/compute/cker/include/cker/Types.h index bb1434f7d4d..c3f04a4d5c8 100644 --- a/compute/cker/include/cker/Types.h +++ b/compute/cker/include/cker/Types.h @@ -63,6 +63,12 @@ enum class ComparisonOpType LessEqual }; +enum class RoPEMode +{ + kGptNeox = 0, + kGptJ = 1, +}; + struct PaddingValues { int16_t width; diff --git a/compute/cker/include/cker/operation/RoPE.h b/compute/cker/include/cker/operation/RoPE.h new file mode 100644 index 00000000000..c000f05af16 --- /dev/null +++ b/compute/cker/include/cker/operation/RoPE.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __NNFW_CKER_ROPE_H__ +#define __NNFW_CKER_ROPE_H__ + +#include "cker/Shape.h" +#include "cker/Types.h" +#include "cker/Utils.h" + +namespace nnfw +{ +namespace cker +{ + +template +inline void RoPE(const RoPEMode mode, const Shape &input_shape, const T *input_data, + const Shape &sin_table_shape, const T *sin_table_data, + const Shape &cos_table_shape, const T *cos_table_data, const Shape &output_shape, + T *output_data) +{ + if (input_shape.Dims(3) != sin_table_shape.Dims(3)) + throw std::runtime_error("the dimension(3) of input and sin_table do not match"); + + if (input_shape.Dims(3) != cos_table_shape.Dims(3)) + throw std::runtime_error("the dimension(3) of input and cos_table do not match"); + + const int32_t i0_n = MatchingDim(input_shape, 0, output_shape, 0); + const int32_t i1_n = MatchingDim(input_shape, 1, output_shape, 1); + const int32_t i2_n = MatchingDim(input_shape, 2, output_shape, 2); + const int32_t i3_n = MatchingDim(input_shape, 3, output_shape, 3); + + if (mode == RoPEMode::kGptNeox) + { + for (int32_t i0 = 0; i0 < i0_n; ++i0) + { + for (int32_t i1 = 0; i1 < i1_n; ++i1) + { + for (int32_t i2 = 0; i2 < i2_n; ++i2) + { + for (int32_t i3 = 0; i3 < i3_n / 2; ++i3) + { + const int32_t offset = Offset(input_shape, i0, i1, i2, i3); + const T x0 = input_data[offset]; + const T x1 = input_data[offset + i3_n / 2]; + + output_data[offset] = x0 * cos_table_data[i3] - x1 * sin_table_data[i3]; + output_data[offset + i3_n / 2] = + x0 * sin_table_data[i3 + i3_n / 2] + x1 * cos_table_data[i3 + i3_n / 2]; + } + } + } + } + } + else + { + throw std::runtime_error("Unsupported RoPE mode"); + } +} + +} // namespace cker +} // namespace nnfw + +#endif // __NNFW_CKER_ROPE_H__ diff --git a/compute/cker/src/RoPE.test.cc b/compute/cker/src/RoPE.test.cc new file mode 100644 index 00000000000..f3866affef7 --- /dev/null +++ b/compute/cker/src/RoPE.test.cc @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "cker/Shape.h" +#include + +#include +#include + +using nnfw::cker::Shape; +using nnfw::cker::RoPEMode; + +TEST(CKer_Operation, RoPE) +{ + // float + { + RoPEMode mode = RoPEMode::kGptNeox; + + Shape input_shape{1, 1, 1, 4}; + std::vector input{0, 1.0, 2.0, 3.0}; + + Shape sin_table_shape{1, 1, 1, 4}; + std::vector sin_table{0.5, 1.0, 1.0, 0.5}; + Shape cos_table_shape{1, 1, 1, 4}; + std::vector cos_table{1.0, 0.5, 0.5, 1.0}; + + Shape ref_output_shape{1, 1, 1, 4}; + std::vector ref_output_data{-1.0, -2.5, 1.0, 3.5}; + + Shape output_shape{1, 1, 1, 4}; + std::vector output(ref_output_data.size()); + + nnfw::cker::RoPE(mode, input_shape, input.data(), sin_table_shape, sin_table.data(), + cos_table_shape, cos_table.data(), ref_output_shape, output.data()); + + for (size_t i = 0; i < ref_output_data.size(); ++i) + { + EXPECT_FLOAT_EQ(ref_output_data[i], output[i]); + } + } + + // int64_t + { + RoPEMode mode = RoPEMode::kGptNeox; + + Shape input_shape{1, 1, 1, 4}; + std::vector input{0, 1, 2, 3}; + + Shape sin_table_shape{1, 1, 1, 4}; + std::vector sin_table{0, 1, 1, 0}; + Shape cos_table_shape{1, 1, 1, 4}; + std::vector cos_table{1, 0, 0, 1}; + + Shape ref_output_shape{1, 1, 1, 4}; + std::vector ref_output_data{0, -3, 0, 3}; + + Shape output_shape{1, 1, 1, 4}; + std::vector output(ref_output_data.size()); + + nnfw::cker::RoPE(mode, input_shape, input.data(), sin_table_shape, sin_table.data(), + cos_table_shape, cos_table.data(), ref_output_shape, output.data()); + + for (size_t i = 0; i < ref_output_data.size(); ++i) + { + EXPECT_EQ(ref_output_data[i], output[i]); + } + } +} + +TEST(CKer_Operation, neg_RoPE) +{ + // the dimension(3) of sin_table and input do not match + { + RoPEMode mode = RoPEMode::kGptNeox; + + Shape input_shape{1, 1, 1, 4}; + std::vector input{0, 1.0, 2.0, 3.0}; + + Shape sin_table_shape{1, 1, 1, 3}; + std::vector sin_table{0.5, 1.0, 1.0}; + Shape cos_table_shape{1, 1, 1, 4}; + std::vector cos_table{1.0, 0.5, 0.5, 1.0}; + + Shape ref_output_shape{1, 1, 1, 4}; + std::vector ref_output_data{-1.0, -2.5, 1.0, 3.5}; + + std::vector output(ref_output_data.size()); + Shape output_shape{1, 1, 1, 4}; + + EXPECT_ANY_THROW(nnfw::cker::RoPE(mode, input_shape, input.data(), sin_table_shape, + sin_table.data(), cos_table_shape, cos_table.data(), + ref_output_shape, output.data())); + } + + // the dimension(3) of cos_table and input do not match + { + RoPEMode mode = RoPEMode::kGptNeox; + + Shape input_shape{1, 1, 1, 4}; + std::vector input{0, 1.0, 2.0, 3.0}; + + Shape sin_table_shape{1, 1, 1, 4}; + std::vector sin_table{0.5, 1.0, 1.0, 0.5}; + Shape cos_table_shape{1, 1, 1, 3}; + std::vector cos_table{1.0, 0.5, 0.5}; + + Shape ref_output_shape{1, 1, 1, 4}; + std::vector ref_output_data{-1.0, -2.5, 1.0, 3.5}; + + std::vector output(ref_output_data.size()); + Shape output_shape{1, 1, 1, 4}; + + EXPECT_ANY_THROW(nnfw::cker::RoPE(mode, input_shape, input.data(), sin_table_shape, + sin_table.data(), cos_table_shape, cos_table.data(), + ref_output_shape, output.data())); + } + + // unsupported RoPE Mode + { + RoPEMode mode = RoPEMode::kGptJ; + + Shape input_shape{1, 1, 1, 4}; + std::vector input{0, 1.0, 2.0, 3.0}; + + Shape sin_table_shape{1, 1, 1, 4}; + std::vector sin_table{0.5, 1.0, 1.0, 0.5}; + Shape cos_table_shape{1, 1, 1, 4}; + std::vector cos_table{1.0, 0.5, 0.5, 1.0}; + + Shape ref_output_shape{1, 1, 1, 4}; + std::vector ref_output_data{-1.0, -2.5, 1.0, 3.5}; + + Shape output_shape{1, 1, 1, 4}; + std::vector output(ref_output_data.size()); + + EXPECT_ANY_THROW(nnfw::cker::RoPE(mode, input_shape, input.data(), sin_table_shape, + sin_table.data(), cos_table_shape, cos_table.data(), + ref_output_shape, output.data())); + } +} diff --git a/runtime/onert/backend/cpu/KernelGenerator.cc b/runtime/onert/backend/cpu/KernelGenerator.cc index e1c6ed5289d..b2ac2e3df53 100644 --- a/runtime/onert/backend/cpu/KernelGenerator.cc +++ b/runtime/onert/backend/cpu/KernelGenerator.cc @@ -49,6 +49,7 @@ #include "ops/ReshapeLayer.h" #include "ops/ResizeBilinearLayer.h" #include "ops/ReverseLayer.h" +#include "ops/RoPELayer.h" #include "ops/SelectLayer.h" #include "ops/ShapeLayer.h" #include "ops/SliceLayer.h" @@ -1543,6 +1544,26 @@ void KernelGenerator::visit(const ir::operation::LSTM &node) _return_fn = std::move(fn); } +void KernelGenerator::visit(const ir::operation::RoPE &node) +{ + const auto input_index{node.getInputs().at(ir::operation::RoPE::Input::INPUT)}; + const auto sin_table{node.getInputs().at(ir::operation::RoPE::Input::SIN_TABLE)}; + const auto cos_table{node.getInputs().at(ir::operation::RoPE::Input::COS_TABLE)}; + const auto output_index{node.getOutputs().at(ir::operation::RoPE::Output::OUTPUT)}; + + auto mode = ops::getRoPEMode(node.param().mode); + + auto input_tensor = _tensor_reg->getPortableTensor(input_index); + auto sin_tensor = _tensor_reg->getPortableTensor(sin_table); + auto cos_tensor = _tensor_reg->getPortableTensor(cos_table); + auto output_tensor = _tensor_reg->getPortableTensor(output_index); + + auto fn = std::make_unique(); + + fn->configure(input_tensor, sin_tensor, cos_tensor, mode, output_tensor); + _return_fn = std::move(fn); +} + } // namespace cpu } // namespace backend } // namespace onert diff --git a/runtime/onert/backend/cpu/KernelGenerator.h b/runtime/onert/backend/cpu/KernelGenerator.h index 8d2010151f8..27c3353c5c2 100644 --- a/runtime/onert/backend/cpu/KernelGenerator.h +++ b/runtime/onert/backend/cpu/KernelGenerator.h @@ -81,6 +81,7 @@ class KernelGenerator : public basic::KernelGeneratorBase void visit(const ir::operation::Reshape &) override; void visit(const ir::operation::ResizeBilinear &node) override; void visit(const ir::operation::Reverse &) override; + void visit(const ir::operation::RoPE &) override; void visit(const ir::operation::Select &) override; void visit(const ir::operation::Shape &) override; void visit(const ir::operation::Slice &) override; diff --git a/runtime/onert/backend/cpu/ops/OperationUtils.cc b/runtime/onert/backend/cpu/ops/OperationUtils.cc index 5e27f175e5f..eef08f4c13a 100644 --- a/runtime/onert/backend/cpu/ops/OperationUtils.cc +++ b/runtime/onert/backend/cpu/ops/OperationUtils.cc @@ -308,6 +308,20 @@ std::vector getReducerAxes(const IPortableTensor *axes) return ret; } +nnfw::cker::RoPEMode getRoPEMode(ir::operation::RoPE::RoPEMode rope_mode) +{ + switch (rope_mode) + { + case ir::operation::RoPE::RoPEMode::GPT_NEOX: + return nnfw::cker::RoPEMode::kGptNeox; + case ir::operation::RoPE::RoPEMode::GPT_J: + return nnfw::cker::RoPEMode::kGptJ; + default: + throw std::runtime_error("Wrong rope mode."); + break; + } +} + } // namespace ops } // namespace cpu } // namespace backend diff --git a/runtime/onert/backend/cpu/ops/OperationUtils.h b/runtime/onert/backend/cpu/ops/OperationUtils.h index 544d07b8785..c354dc49bab 100644 --- a/runtime/onert/backend/cpu/ops/OperationUtils.h +++ b/runtime/onert/backend/cpu/ops/OperationUtils.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -169,6 +170,8 @@ nnfw::cker::PaddingType getPaddingType(ir::PaddingType ir_padding_type); std::vector getReducerAxes(const IPortableTensor *axes); +nnfw::cker::RoPEMode getRoPEMode(ir::operation::RoPE::RoPEMode rope_mode); + template const T *getBuffer(const IPortableTensor *tensor) { return reinterpret_cast(tensor->buffer()); diff --git a/runtime/onert/backend/cpu/ops/RoPELayer.cc b/runtime/onert/backend/cpu/ops/RoPELayer.cc new file mode 100644 index 00000000000..7e3f7d84dbb --- /dev/null +++ b/runtime/onert/backend/cpu/ops/RoPELayer.cc @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "RoPELayer.h" + +#include + +namespace onert +{ +namespace backend +{ +namespace cpu +{ +namespace ops +{ + +RoPELayer::RoPELayer() + : _input(nullptr), _sin(nullptr), _cos(nullptr), _mode(nnfw::cker::RoPEMode::kGptNeox), + _output(nullptr) +{ + // DO NOTHING +} + +RoPELayer::~RoPELayer() = default; + +void RoPELayer::configure(const IPortableTensor *input, const IPortableTensor *sin, + const IPortableTensor *cos, nnfw::cker::RoPEMode mode, + IPortableTensor *output) +{ + assert(input != nullptr); + assert(sin != nullptr); + assert(cos != nullptr); + assert(output != nullptr); + + _input = input; + _sin = sin; + _cos = cos; + _mode = mode; + _output = output; +} + +template void RoPELayer::rope() +{ + auto input_shape = _input->getShape(); + assert(input_shape.rank() == 4); + + nnfw::cker::RoPE(_mode, getShape(_input), getBuffer(_input), getShape(_sin), + getBuffer(_sin), getShape(_cos), getBuffer(_cos), getShape(_output), + getBuffer(_output)); +} + +void RoPELayer::run() +{ + switch (_input->data_type()) + { + case OperandType::FLOAT32: + rope(); + break; + case OperandType::INT32: + rope(); + break; + case OperandType::INT64: + rope(); + break; + case OperandType::QUANT_UINT8_ASYMM: + rope(); + break; + case OperandType::QUANT_INT8_ASYMM: + rope(); + break; + default: + throw std::runtime_error("RoPE: unsupported data type"); + } +} + +} // namespace ops +} // namespace cpu +} // namespace backend +} // namespace onert diff --git a/runtime/onert/backend/cpu/ops/RoPELayer.h b/runtime/onert/backend/cpu/ops/RoPELayer.h new file mode 100644 index 00000000000..d51d9ae9069 --- /dev/null +++ b/runtime/onert/backend/cpu/ops/RoPELayer.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_BACKEND_CPU_OPS_ROPE_LAYER_H__ +#define __ONERT_BACKEND_CPU_OPS_ROPE_LAYER_H__ + +#include +#include "OperationUtils.h" + +#include + +namespace nnfw +{ +namespace cker +{ +class RoPE; +} +} // namespace nnfw + +namespace onert +{ +namespace backend +{ +namespace cpu +{ +namespace ops +{ + +class RoPELayer : public ::onert::exec::IFunction +{ +public: + RoPELayer(); + ~RoPELayer(); + +public: + template void rope(); + + void configure(const IPortableTensor *input, const IPortableTensor *sin, + const IPortableTensor *cos, const nnfw::cker::RoPEMode mode, + IPortableTensor *output); + + void run() override; + +private: + const IPortableTensor *_input; + const IPortableTensor *_sin; + const IPortableTensor *_cos; + + nnfw::cker::RoPEMode _mode; + IPortableTensor *_output; +}; + +} // namespace ops +} // namespace cpu +} // namespace backend +} // namespace onert + +#endif // __ONERT_BACKEND_CPU_OPS_ROPE_LAYER_H__ diff --git a/runtime/onert/core/include/ir/Operations.Include.h b/runtime/onert/core/include/ir/Operations.Include.h index 6352b8ed90b..b5ef8d52970 100644 --- a/runtime/onert/core/include/ir/Operations.Include.h +++ b/runtime/onert/core/include/ir/Operations.Include.h @@ -67,6 +67,7 @@ #include "ir/operation/ResizeBilinear.h" #include "ir/operation/ResizeNearestNeighbor.h" #include "ir/operation/Reverse.h" +#include "ir/operation/RoPE.h" #include "ir/operation/RNN.h" #include "ir/operation/Select.h" #include "ir/operation/Shape.h" diff --git a/runtime/onert/core/include/ir/Operations.lst b/runtime/onert/core/include/ir/Operations.lst index 1f91aecb23f..46f61acecbd 100644 --- a/runtime/onert/core/include/ir/Operations.lst +++ b/runtime/onert/core/include/ir/Operations.lst @@ -70,6 +70,7 @@ OP(ResizeBilinear) OP(ResizeNearestNeighbor) OP(Reverse) OP(RNN) +OP(RoPE) OP(Select) OP(Shape) OP(Slice) diff --git a/runtime/onert/core/include/ir/operation/RoPE.h b/runtime/onert/core/include/ir/operation/RoPE.h new file mode 100644 index 00000000000..85984213f4a --- /dev/null +++ b/runtime/onert/core/include/ir/operation/RoPE.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __ONERT_IR_OPERATION_ROPE_H__ +#define __ONERT_IR_OPERATION_ROPE_H__ + +#include "ir/Operation.h" +#include "ir/InternalType.h" + +namespace onert +{ +namespace ir +{ +namespace operation +{ + +class RoPE : public Operation +{ +public: + enum Input + { + INPUT = 0, + SIN_TABLE, + COS_TABLE + }; + + enum Output + { + OUTPUT = 0, + }; + + enum class RoPEMode + { + GPT_NEOX = 0, + GPT_J = 1, + }; + + struct Param + { + RoPEMode mode; + }; + +public: + RoPE(const OperandIndexSequence &inputs, const OperandIndexSequence &outputs, const Param ¶m); + +public: + void accept(OperationVisitor &v) const override; + OpCode opcode() const final { return OpCode::RoPE; } + +public: + const Param ¶m() const { return _param; } + +private: + Param _param; +}; + +} // namespace operation +} // namespace ir +} // namespace onert + +#endif // __ONERT_IR_OPERATION_ROPE_H__ diff --git a/runtime/onert/core/include/loader/CircleLoader.h b/runtime/onert/core/include/loader/CircleLoader.h index a65fa09b046..de8af536bda 100644 --- a/runtime/onert/core/include/loader/CircleLoader.h +++ b/runtime/onert/core/include/loader/CircleLoader.h @@ -18,6 +18,7 @@ #define __ONERT_LOADER_CIRCLE_LOADER_H__ #include "ir/Graph.h" +#include "ir/operation/RoPE.h" #include diff --git a/runtime/onert/core/src/ir/OperationValidator.cc b/runtime/onert/core/src/ir/OperationValidator.cc index 3aa94d5f8d6..6f08161b99d 100644 --- a/runtime/onert/core/src/ir/OperationValidator.cc +++ b/runtime/onert/core/src/ir/OperationValidator.cc @@ -451,6 +451,19 @@ void OperationValidator::visit(const operation::Reverse &node) OP_REQUIRES(isSameType(output_index, input_index)); } +void OperationValidator::visit(const operation::RoPE &node) +{ + const auto input_index{node.getInputs().at(operation::RoPE::Input::INPUT)}; + const auto sin_index{node.getInputs().at(operation::RoPE::Input::SIN_TABLE)}; + const auto cos_index{node.getInputs().at(operation::RoPE::Input::COS_TABLE)}; + const auto output_index{node.getOutputs().at(operation::RoPE::Output::OUTPUT)}; + + OP_REQUIRES(isValidType(input_index, DataType::FLOAT32)); + OP_REQUIRES(isValidType(sin_index, DataType::FLOAT32)); + OP_REQUIRES(isValidType(cos_index, DataType::FLOAT32)); + OP_REQUIRES(isSameType(input_index, output_index)); +} + void OperationValidator::visit(const operation::Select &node) { const auto condition_index{node.getInputs().at(operation::Select::Input::CONDITION)}; diff --git a/runtime/onert/core/src/ir/OperationValidator.h b/runtime/onert/core/src/ir/OperationValidator.h index f5116e24269..79b4166a9be 100644 --- a/runtime/onert/core/src/ir/OperationValidator.h +++ b/runtime/onert/core/src/ir/OperationValidator.h @@ -69,6 +69,7 @@ class OperationValidator : public OperationVisitor void visit(const operation::Rank &node) override; void visit(const operation::ResizeBilinear &node) override; void visit(const operation::Reverse &node) override; + void visit(const operation::RoPE &node) override; void visit(const operation::Select &node) override; void visit(const operation::Shape &node) override; void visit(const operation::Slice &node) override; diff --git a/runtime/onert/core/src/ir/operation/RoPE.cc b/runtime/onert/core/src/ir/operation/RoPE.cc new file mode 100644 index 00000000000..c03094671a8 --- /dev/null +++ b/runtime/onert/core/src/ir/operation/RoPE.cc @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/operation/RoPE.h" +#include "ir/OperationVisitor.h" + +namespace onert +{ +namespace ir +{ +namespace operation +{ + +void RoPE::accept(OperationVisitor &v) const { v.visit(*this); } + +RoPE::RoPE(const OperandIndexSequence &inputs, const OperandIndexSequence &outputs, + const Param ¶m) + : Operation{OperandConstraint::createExact(3u), inputs, outputs}, _param{param} +{ +} + +} // namespace operation +} // namespace ir +} // namespace onert diff --git a/runtime/onert/core/src/loader/CircleLoader.cc b/runtime/onert/core/src/loader/CircleLoader.cc index da5b4b892be..d1897a3200e 100644 --- a/runtime/onert/core/src/loader/CircleLoader.cc +++ b/runtime/onert/core/src/loader/CircleLoader.cc @@ -70,6 +70,7 @@ class CircleLoader final : public loader::BaseLoader void loadInstanceNorm(const Operator *op, ir::Graph &subg); void loadBCQFullyConnected(const Operator *op, ir::Graph &subg); void loadBCQGather(const Operator *op, ir::Graph &subg); + void loadRoPE(const Operator *op, ir::Graph &subg); public: using BaseLoader::BaseLoader; @@ -98,6 +99,20 @@ class CircleLoader final : public loader::BaseLoader return BaseLoader::tensorTypeToDataType(type); } + ir::operation::RoPE::RoPEMode convertRoPEMode(const circle::RoPEMode mode) + { + switch (mode) + { + case circle::RoPEMode::RoPEMode_GPT_NEOX: + return ir::operation::RoPE::RoPEMode::GPT_NEOX; + case circle::RoPEMode::RoPEMode_GPT_J: + return ir::operation::RoPE::RoPEMode::GPT_J; + default: + throw std::runtime_error(std::string("Unsupported RoPE mode: ") + + std::to_string(static_cast(mode))); + } + } + private: std::unique_ptr loadSubgraph(const circle::SubGraph *circle_subg) override { @@ -150,6 +165,9 @@ class CircleLoader final : public loader::BaseLoader case circle::BuiltinOperator::BuiltinOperator_BCQ_GATHER: loadBCQGather(op, subg); return; + case circle::BuiltinOperator::BuiltinOperator_ROPE: + loadRoPE(op, subg); + return; default: BaseLoader::loadOperation(op, subg); return; @@ -225,6 +243,22 @@ void CircleLoader::loadBCQFullyConnected(const Operator *op, ir::Graph &subg) subg.addOperation(std::move(new_op)); } +void CircleLoader::loadRoPE(const Operator *op, ir::Graph &subg) +{ + ir::OperandIndexSequence inputs; + ir::OperandIndexSequence outputs; + + loadOperationIO(op, inputs, outputs); + + ir::operation::RoPE::Param param; + const auto *options = op->builtin_options_as_RoPEOptions(); + + param.mode = convertRoPEMode(options->mode()); + + std::unique_ptr new_op(new ir::operation::RoPE(inputs, outputs, param)); + subg.addOperation(std::move(new_op)); +} + } // namespace std::unique_ptr loadCircleModel(const std::string &filename) diff --git a/tests/nnfw_api/lib/CircleGen.cc b/tests/nnfw_api/lib/CircleGen.cc index b73ac8f65e8..2fc943dd41f 100644 --- a/tests/nnfw_api/lib/CircleGen.cc +++ b/tests/nnfw_api/lib/CircleGen.cc @@ -589,6 +589,13 @@ uint32_t CircleGen::addOperatorBatchToSpaceND(const OperatorParams ¶ms) circle::BuiltinOptions_BatchToSpaceNDOptions, options); } +uint32_t CircleGen::addOperatorRoPE(const OperatorParams ¶ms) +{ + auto options = circle::CreateRoPEOptions(_fbb).Union(); + return addOperatorWithOptions(params, circle::BuiltinOperator_ROPE, + circle::BuiltinOptions_RoPEOptions, options); +} + // NOTE Please add addOperator functions ABOVE this lie // // % How to add a new addOperatorXXX fuction diff --git a/tests/nnfw_api/lib/CircleGen.h b/tests/nnfw_api/lib/CircleGen.h index 72ce6a4e9c4..f7336f54a5d 100644 --- a/tests/nnfw_api/lib/CircleGen.h +++ b/tests/nnfw_api/lib/CircleGen.h @@ -182,6 +182,7 @@ class CircleGen uint32_t addOperatorIf(const OperatorParams ¶ms, uint32_t then_subg, uint32_t else_subg); uint32_t addOperatorInstanceNorm(const OperatorParams ¶ms, float epsilon, circle::ActivationFunctionType actfn); + uint32_t addOperatorRoPE(const OperatorParams ¶ms); uint32_t addOperatorL2Normalization(const OperatorParams ¶ms); uint32_t addOperatorLeakyRelu(const OperatorParams ¶ms, float alpha); uint32_t addOperatorLess(const OperatorParams ¶ms);