From d0046f2f5a49a84677303f627dbf782a0047798d Mon Sep 17 00:00:00 2001
From: AsakusaRinne <AsakusaRinne@gmail.com>
Date: Thu, 27 Oct 2022 14:31:35 +0800
Subject: [PATCH 1/5] create ternary-generic design

---
 .gitignore                                    |  45 +++
 Benchmark/Benchmark.csproj                    |  20 ++
 Benchmark/Program.cs                          |  39 +++
 NN.BenchmarkTest/MatmulBenchmark.cs           |  64 ++++
 NN.BenchmarkTest/NN.BenchmarkTest.csproj      |  21 ++
 NN.BenchmarkTest/Program.cs                   |   4 +
 NN.CodeGen/NN.CodeGen.csproj                  |  10 +
 NN.CodeGen/Program.cs                         |   2 +
 NN.Core/Class1.cs                             |   7 +
 NN.Core/NN.Core.csproj                        |  13 +
 NN.UnitTests/NN.UnitTests.csproj              |  28 ++
 NN.UnitTests/NaiveTest.cs                     |  43 +++
 NN.UnitTests/Usings.cs                        |   1 +
 NN.sln                                        |  55 ++++
 NN/Abstraction/DType/IBinaryDTypeHandler.cs   |  34 +++
 NN/Abstraction/DType/INativeConvertible.cs    |  13 +
 NN/Abstraction/DType/INativeDType.cs          |  15 +
 NN/Abstraction/DType/ITernaryDTypeHandler.cs  |  54 ++++
 NN/Abstraction/DType/IUnaryDTypeHandler.cs    |  18 ++
 NN/Abstraction/Data/INativeMemoryManager.cs   |  13 +
 NN/Abstraction/Operators/IBinaryOperator.cs   |   9 +
 NN/Abstraction/Operators/IMatmulOperator.cs   |   9 +
 NN/Abstraction/Operators/IOperator.cs         |  14 +
 NN/Abstraction/Operators/ITernaryOperator.cs  |   9 +
 .../Operators/ITypeConvertOperator.cs         |  11 +
 NN/Abstraction/Operators/IUnaryOperator.cs    |   9 +
 .../Operators/OperatorHandlerType.cs          |  16 +
 NN/Basic/DType/BoolHandler.cs                 |  38 +++
 NN/Basic/DType/FloatHandler.cs                |  43 +++
 NN/Basic/DType/Int32Handler.cs                |  39 +++
 NN/Basic/NativeLayout.cs                      | 287 ++++++++++++++++++
 NN/Data/DefaultNativeMemoryManager.cs         |  20 ++
 NN/Data/NativeArray.cs                        |  74 +++++
 NN/Data/NativeStorage.cs                      |  45 +++
 NN/Extensions/IndexOffsetConvertExtension.cs  |  34 +++
 NN/Extensions/PrintArrayExtension.cs          |  68 +++++
 NN/NN.Native.csproj                           |  15 +
 NN/Operators/Common/Params/MatmulParam.cs     |  14 +
 .../Common/Params/TypeConvertParam.cs         |  15 +
 .../Common/TypeConvertOperatorBase.cs         |  19 ++
 NN/Operators/Naive/MatmulOperator.cs          |  35 +++
 NN/Operators/Naive/TypeConvertOperator.cs     |  40 +++
 42 files changed, 1362 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 Benchmark/Benchmark.csproj
 create mode 100644 Benchmark/Program.cs
 create mode 100644 NN.BenchmarkTest/MatmulBenchmark.cs
 create mode 100644 NN.BenchmarkTest/NN.BenchmarkTest.csproj
 create mode 100644 NN.BenchmarkTest/Program.cs
 create mode 100644 NN.CodeGen/NN.CodeGen.csproj
 create mode 100644 NN.CodeGen/Program.cs
 create mode 100644 NN.Core/Class1.cs
 create mode 100644 NN.Core/NN.Core.csproj
 create mode 100644 NN.UnitTests/NN.UnitTests.csproj
 create mode 100644 NN.UnitTests/NaiveTest.cs
 create mode 100644 NN.UnitTests/Usings.cs
 create mode 100644 NN.sln
 create mode 100644 NN/Abstraction/DType/IBinaryDTypeHandler.cs
 create mode 100644 NN/Abstraction/DType/INativeConvertible.cs
 create mode 100644 NN/Abstraction/DType/INativeDType.cs
 create mode 100644 NN/Abstraction/DType/ITernaryDTypeHandler.cs
 create mode 100644 NN/Abstraction/DType/IUnaryDTypeHandler.cs
 create mode 100644 NN/Abstraction/Data/INativeMemoryManager.cs
 create mode 100644 NN/Abstraction/Operators/IBinaryOperator.cs
 create mode 100644 NN/Abstraction/Operators/IMatmulOperator.cs
 create mode 100644 NN/Abstraction/Operators/IOperator.cs
 create mode 100644 NN/Abstraction/Operators/ITernaryOperator.cs
 create mode 100644 NN/Abstraction/Operators/ITypeConvertOperator.cs
 create mode 100644 NN/Abstraction/Operators/IUnaryOperator.cs
 create mode 100644 NN/Abstraction/Operators/OperatorHandlerType.cs
 create mode 100644 NN/Basic/DType/BoolHandler.cs
 create mode 100644 NN/Basic/DType/FloatHandler.cs
 create mode 100644 NN/Basic/DType/Int32Handler.cs
 create mode 100644 NN/Basic/NativeLayout.cs
 create mode 100644 NN/Data/DefaultNativeMemoryManager.cs
 create mode 100644 NN/Data/NativeArray.cs
 create mode 100644 NN/Data/NativeStorage.cs
 create mode 100644 NN/Extensions/IndexOffsetConvertExtension.cs
 create mode 100644 NN/Extensions/PrintArrayExtension.cs
 create mode 100644 NN/NN.Native.csproj
 create mode 100644 NN/Operators/Common/Params/MatmulParam.cs
 create mode 100644 NN/Operators/Common/Params/TypeConvertParam.cs
 create mode 100644 NN/Operators/Common/TypeConvertOperatorBase.cs
 create mode 100644 NN/Operators/Naive/MatmulOperator.cs
 create mode 100644 NN/Operators/Naive/TypeConvertOperator.cs

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..a2aa9a1
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,45 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+#builds
+build/
+
+#developments
+.vscode/
+
+#CSharp
+.vs/
+bin/
+obj/
+*/bin/
+*/obj/
\ No newline at end of file
diff --git a/Benchmark/Benchmark.csproj b/Benchmark/Benchmark.csproj
new file mode 100644
index 0000000..612e80d
--- /dev/null
+++ b/Benchmark/Benchmark.csproj
@@ -0,0 +1,20 @@
+﻿<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
+    <LangVersion>preview</LangVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\NN\NN.Native.csproj" />
+  </ItemGroup>
+
+</Project>
diff --git a/Benchmark/Program.cs b/Benchmark/Program.cs
new file mode 100644
index 0000000..643936f
--- /dev/null
+++ b/Benchmark/Program.cs
@@ -0,0 +1,39 @@
+﻿using BenchmarkDotNet.Running;
+using NN.Native.Operators;
+using BenchmarkDotNet.Attributes;
+using System.Reflection;
+using NN.Native.Data;
+using NN.Native.Basic;
+using NN.Native.Operators.Naive;
+using NN.Native.Basic.DType;
+
+var summary = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
+Console.WriteLine(summary);
+
+public class MatmulFloat32Benchmark
+{
+    public IEnumerable<int[][]> DimensionValues => new int[][][] {
+            new int[][]{ new int[] { 3, 4 }, new int[] { 4, 5} },
+            new int[][]{ new int[] { 32, 64 }, new int[] { 64, 48} },
+            new int[][]{ new int[] { 256, 512 }, new int[] { 512, 192} },
+            new int[][]{ new int[] { 8, 1024 }, new int[] { 1024, 1} },
+        };
+    [ParamsSource(nameof(DimensionValues))]
+    public int[][] Dimensions { get; set; }
+
+    NativeArray<float> ArrayLeft { get; set; }
+    NativeArray<float> ArrayRight { get; set; }
+    [GlobalSetup(Targets = new[] { nameof(TensornetMatmul) })]
+    public void TensornetSetup()
+    {
+        ArrayLeft = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[0]), -100, 100);
+        ArrayRight = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[1]), -100, 100);
+    }
+
+    [Benchmark]
+    public void TensornetMatmul()
+    {
+        //NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
+        //MatmulOperator<float, float, float, FloatHandler>.Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+    }
+}
\ No newline at end of file
diff --git a/NN.BenchmarkTest/MatmulBenchmark.cs b/NN.BenchmarkTest/MatmulBenchmark.cs
new file mode 100644
index 0000000..1fe773d
--- /dev/null
+++ b/NN.BenchmarkTest/MatmulBenchmark.cs
@@ -0,0 +1,64 @@
+﻿using NN.Native.Operators;
+using BenchmarkDotNet.Attributes;
+using System.Reflection;
+using NN.Native.Data;
+using NN.Native.Basic;
+using static Tensorflow.Binding;
+using NN.Native.Operators.Naive;
+using NN.Native.Basic.DType;
+
+[MemoryDiagnoser]
+[BenchmarkCategory("MatmulFloat32")]
+public class MatmulFloat32Benchmark
+{
+    public IEnumerable<int[][]> DimensionValues => new int[][][] {
+            new int[][]{ new int[] { 3, 4 }, new int[] { 4, 5} },
+            new int[][]{ new int[] { 32, 64 }, new int[] { 64, 48} },
+            new int[][]{ new int[] { 256, 512 }, new int[] { 512, 192} },
+            new int[][]{ new int[] { 8, 1024 }, new int[] { 1024, 1} },
+        };
+    [ParamsSource(nameof(DimensionValues))]
+    public int[][] Dimensions { get; set; }
+
+    NativeArray<float> ArrayLeft { get; set; }
+    NativeArray<float> ArrayRight { get; set; }
+    NumSharp.NDArray NDArrayLeft { get; set; }
+    NumSharp.NDArray NDArrayRight { get; set; }
+    Tensorflow.NumPy.NDArray TFNetTensorLeft { get; set; }
+    Tensorflow.NumPy.NDArray TFNetTensorRight { get; set; }
+    [GlobalSetup(Targets = new[] { nameof(TensornetMatmul) })]
+    public void TensornetSetup()
+    {
+        ArrayLeft = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[0]), -100, 100);
+        ArrayRight = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[1]), -100, 100);
+    }
+    [GlobalSetup(Targets = new[] { nameof(NumSharpMatmul) })]
+    public void NumSharpSetup()
+    {
+        NDArrayLeft = NumSharp.np.random.uniform(-100, 100, new NumSharp.Shape(Dimensions[0]));
+        NDArrayRight = NumSharp.np.random.uniform(-100, 100, new NumSharp.Shape(Dimensions[1]));
+    }
+    [GlobalSetup(Targets = new[] { nameof(TFNetMatmul) })]
+    public void TFNetSetup()
+    {
+        TFNetTensorLeft = Tensorflow.NumPy.np.random.normal(0, 100, new Tensorflow.Shape(Dimensions[0]));
+        TFNetTensorRight = Tensorflow.NumPy.np.random.normal(0, 100, new Tensorflow.Shape(Dimensions[1]));
+    }
+
+    [Benchmark]
+    public void TensornetMatmul()
+    {
+        NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
+        MatmulOperator<float, float, float, FloatHandler>.Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+    }
+    [Benchmark]
+    public void NumSharpMatmul()
+    {
+        NumSharp.np.matmul(NDArrayLeft, NDArrayRight);
+    }
+    [Benchmark]
+    public void TFNetMatmul()
+    {
+        tf.matmul(TFNetTensorLeft, TFNetTensorRight);
+    }
+}
\ No newline at end of file
diff --git a/NN.BenchmarkTest/NN.BenchmarkTest.csproj b/NN.BenchmarkTest/NN.BenchmarkTest.csproj
new file mode 100644
index 0000000..1bd6885
--- /dev/null
+++ b/NN.BenchmarkTest/NN.BenchmarkTest.csproj
@@ -0,0 +1,21 @@
+﻿<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.2" />
+    <PackageReference Include="NumSharp" Version="0.30.0" />
+    <PackageReference Include="SciSharp.TensorFlow.Redist" Version="2.7.0" />
+    <PackageReference Include="TensorFlow.NET" Version="0.70.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\NN\NN.Native.csproj" />
+  </ItemGroup>
+
+</Project>
diff --git a/NN.BenchmarkTest/Program.cs b/NN.BenchmarkTest/Program.cs
new file mode 100644
index 0000000..c81bdf8
--- /dev/null
+++ b/NN.BenchmarkTest/Program.cs
@@ -0,0 +1,4 @@
+﻿using BenchmarkDotNet.Running;
+
+var summary = BenchmarkRunner.Run(typeof(Program).Assembly);
+Console.WriteLine(summary);
\ No newline at end of file
diff --git a/NN.CodeGen/NN.CodeGen.csproj b/NN.CodeGen/NN.CodeGen.csproj
new file mode 100644
index 0000000..f02677b
--- /dev/null
+++ b/NN.CodeGen/NN.CodeGen.csproj
@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>
diff --git a/NN.CodeGen/Program.cs b/NN.CodeGen/Program.cs
new file mode 100644
index 0000000..3751555
--- /dev/null
+++ b/NN.CodeGen/Program.cs
@@ -0,0 +1,2 @@
+﻿// See https://aka.ms/new-console-template for more information
+Console.WriteLine("Hello, World!");
diff --git a/NN.Core/Class1.cs b/NN.Core/Class1.cs
new file mode 100644
index 0000000..478914c
--- /dev/null
+++ b/NN.Core/Class1.cs
@@ -0,0 +1,7 @@
+﻿namespace NN.Core
+{
+    public class Class1
+    {
+
+    }
+}
\ No newline at end of file
diff --git a/NN.Core/NN.Core.csproj b/NN.Core/NN.Core.csproj
new file mode 100644
index 0000000..496b1e7
--- /dev/null
+++ b/NN.Core/NN.Core.csproj
@@ -0,0 +1,13 @@
+﻿<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\NN\NN.Native.csproj" />
+  </ItemGroup>
+
+</Project>
diff --git a/NN.UnitTests/NN.UnitTests.csproj b/NN.UnitTests/NN.UnitTests.csproj
new file mode 100644
index 0000000..9631cd9
--- /dev/null
+++ b/NN.UnitTests/NN.UnitTests.csproj
@@ -0,0 +1,28 @@
+﻿<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\NN\NN.Native.csproj" />
+  </ItemGroup>
+
+</Project>
diff --git a/NN.UnitTests/NaiveTest.cs b/NN.UnitTests/NaiveTest.cs
new file mode 100644
index 0000000..6cbf209
--- /dev/null
+++ b/NN.UnitTests/NaiveTest.cs
@@ -0,0 +1,43 @@
+using NN.Native.Data;
+using NN.Native.Extensions;
+using NN.Native.Basic;
+using NN.Native.Operators.Naive;
+using NN.Native.Basic.DType;
+using NN.Native.Operators.Common.Params;
+using Xunit.Abstractions;
+
+namespace NN.UnitTests
+{
+    public class NaiveTest
+    {
+        private readonly ITestOutputHelper _output;
+
+        public NaiveTest(ITestOutputHelper output)
+        {
+            this._output = output;
+        }
+        [Fact]
+        public void TypeConvertTest()
+        {
+            var array = NativeArray.FromArray(new NativeLayout(new int[] { 3, 4 }), new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 });
+            _output.WriteLine(array.Print());
+            var convertedArray = new NativeArray<float>(new NativeLayout(new int[] { 4, 3 }), new DefaultNativeMemoryManager());
+            TypeConvertOperator<int, float, FloatHandler>.Exec(array.Span, convertedArray.Span, array._layout, convertedArray._layout, new TypeConvertParam()
+            {
+                Transpose = true,
+                DimA = 0,
+                DimB = 1
+            });
+            _output.WriteLine(convertedArray.Print());
+        }
+        [Fact]
+        public void MatmulTest()
+        {
+            var a = NativeArray.FromArray(new NativeLayout(new int[] { 3, 4 }), new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 });
+            var b = NativeArray.FromArray(new NativeLayout(new int[] { 4, 2 }), new int[] { 1, 2, 1, 2, 1, 1, 2, 3 });
+            var c = new NativeArray<int>(new NativeLayout(new int[] { 3, 2 }), new DefaultNativeMemoryManager());
+            MatmulOperator<int, int, int, Int32Handler>.Exec(a.Span, b.Span, c.Span, a._layout, b._layout, c._layout);
+            _output.WriteLine(c.Print());
+        }
+    }
+}
\ No newline at end of file
diff --git a/NN.UnitTests/Usings.cs b/NN.UnitTests/Usings.cs
new file mode 100644
index 0000000..8c927eb
--- /dev/null
+++ b/NN.UnitTests/Usings.cs
@@ -0,0 +1 @@
+global using Xunit;
\ No newline at end of file
diff --git a/NN.sln b/NN.sln
new file mode 100644
index 0000000..e3b9761
--- /dev/null
+++ b/NN.sln
@@ -0,0 +1,55 @@
+﻿
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32922.545
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NN.Native", "NN\NN.Native.csproj", "{10731AE4-51D6-404A-A971-CF5D8074CFAC}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NN.CodeGen", "NN.CodeGen\NN.CodeGen.csproj", "{B11B48A2-9395-4E2D-BC72-26FC51F15736}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NN.UnitTests", "NN.UnitTests\NN.UnitTests.csproj", "{87929F21-15B4-4903-ADB8-BD5E8265E027}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NN.BenchmarkTest", "NN.BenchmarkTest\NN.BenchmarkTest.csproj", "{F70240F1-62F2-4F6B-9958-1240A3A40971}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NN.Core", "NN.Core\NN.Core.csproj", "{C61FC3B6-FD8C-4BDA-BC69-AB4BD46C8867}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmark", "Benchmark\Benchmark.csproj", "{9A84C621-DF8A-4D37-B980-F8DCC4638B40}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{10731AE4-51D6-404A-A971-CF5D8074CFAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{10731AE4-51D6-404A-A971-CF5D8074CFAC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{10731AE4-51D6-404A-A971-CF5D8074CFAC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{10731AE4-51D6-404A-A971-CF5D8074CFAC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B11B48A2-9395-4E2D-BC72-26FC51F15736}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B11B48A2-9395-4E2D-BC72-26FC51F15736}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B11B48A2-9395-4E2D-BC72-26FC51F15736}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B11B48A2-9395-4E2D-BC72-26FC51F15736}.Release|Any CPU.Build.0 = Release|Any CPU
+		{87929F21-15B4-4903-ADB8-BD5E8265E027}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{87929F21-15B4-4903-ADB8-BD5E8265E027}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{87929F21-15B4-4903-ADB8-BD5E8265E027}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{87929F21-15B4-4903-ADB8-BD5E8265E027}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F70240F1-62F2-4F6B-9958-1240A3A40971}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F70240F1-62F2-4F6B-9958-1240A3A40971}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F70240F1-62F2-4F6B-9958-1240A3A40971}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F70240F1-62F2-4F6B-9958-1240A3A40971}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C61FC3B6-FD8C-4BDA-BC69-AB4BD46C8867}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C61FC3B6-FD8C-4BDA-BC69-AB4BD46C8867}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C61FC3B6-FD8C-4BDA-BC69-AB4BD46C8867}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C61FC3B6-FD8C-4BDA-BC69-AB4BD46C8867}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9A84C621-DF8A-4D37-B980-F8DCC4638B40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9A84C621-DF8A-4D37-B980-F8DCC4638B40}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9A84C621-DF8A-4D37-B980-F8DCC4638B40}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9A84C621-DF8A-4D37-B980-F8DCC4638B40}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {D258A986-B952-45D9-B355-DC6AB13833F4}
+	EndGlobalSection
+EndGlobal
diff --git a/NN/Abstraction/DType/IBinaryDTypeHandler.cs b/NN/Abstraction/DType/IBinaryDTypeHandler.cs
new file mode 100644
index 0000000..8d4f2d4
--- /dev/null
+++ b/NN/Abstraction/DType/IBinaryDTypeHandler.cs
@@ -0,0 +1,34 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.DType
+{
+    public unsafe interface IBinaryDTypeHandler<TA, TB> where TA : unmanaged where TB : unmanaged
+    {
+        static abstract void AddInplace(ref TA src, in TB addend);
+        static abstract void SubstractInplace(ref TA src, in TB subtrahend);
+        static abstract void MultiplyInplace(ref TA src, in TB multiplicator);
+        static abstract void DivideInplace(ref TA src, in TB divisor);
+        static abstract void ModuleInplace(ref TA src, in TB modulus);
+        //static abstract void LeftShift(ref TA src, ref TB dst, int offset);
+        //static abstract void RightShift(ref TA src, ref TB dst, int offset);
+        static abstract bool AreEqual (in TA a, in TB b);
+        /// <summary>
+        /// Return if a is greater than b.
+        /// </summary>
+        /// <param name="a"></param>
+        /// <param name="b"></param>
+        /// <returns></returns>
+        static abstract bool IsGraeter(in TA a, in TB b);
+        /// <summary>
+        /// Return if a is less than b.
+        /// </summary>
+        /// <param name="a"></param>
+        /// <param name="b"></param>
+        /// <returns></returns>
+        static abstract bool IsLess(in TA a, in TB b);
+    }
+}
diff --git a/NN/Abstraction/DType/INativeConvertible.cs b/NN/Abstraction/DType/INativeConvertible.cs
new file mode 100644
index 0000000..65c84a4
--- /dev/null
+++ b/NN/Abstraction/DType/INativeConvertible.cs
@@ -0,0 +1,13 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.DType
+{
+    public interface INativeConvertible<TA, TB> where TA: unmanaged where TB: unmanaged
+    {
+        static abstract void Convert(in TA a, ref TB b);
+    }
+}
diff --git a/NN/Abstraction/DType/INativeDType.cs b/NN/Abstraction/DType/INativeDType.cs
new file mode 100644
index 0000000..eb9a2be
--- /dev/null
+++ b/NN/Abstraction/DType/INativeDType.cs
@@ -0,0 +1,15 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.DType
+{
+    //internal interface INativeDTypeHandler<TA, TB, TC>: 
+    //    IBinaryDTypeHandler<TA, TC>, IBinaryDTypeHandler<TB, TC>, ITernaryDTypeHandler<TA, TB, TC>
+    //    where TA: unmanaged where TB: unmanaged where TC: unmanaged
+    //{
+        
+    //}
+}
diff --git a/NN/Abstraction/DType/ITernaryDTypeHandler.cs b/NN/Abstraction/DType/ITernaryDTypeHandler.cs
new file mode 100644
index 0000000..0bbf4f9
--- /dev/null
+++ b/NN/Abstraction/DType/ITernaryDTypeHandler.cs
@@ -0,0 +1,54 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.DType
+{
+    public interface ITernaryDTypeHandler<TA, TB, TC> where TA : unmanaged where TB : unmanaged where TC: unmanaged
+    {
+        /// <summary>
+        /// dst = a + b
+        /// </summary>
+        /// <param name="a"></param>
+        /// <param name="b"></param>
+        /// <param name="dst"></param>
+        static abstract void Add(in TA a, in TB b, ref TC dst);
+        /// <summary>
+        /// dst = a - b
+        /// </summary>
+        /// <param name="a"></param>
+        /// <param name="b"></param>
+        /// <param name="dst"></param>
+        static abstract void Substract(in TA a, in TB b, ref TC dst);
+        /// <summary>
+        /// dst = a * b
+        /// </summary>
+        /// <param name="a"></param>
+        /// <param name="b"></param>
+        /// <param name="dst"></param>
+        static abstract void Multiply(in TA a, in TB b, ref TC dst);
+        /// <summary>
+        /// dst = a / b
+        /// </summary>
+        /// <param name="a"></param>
+        /// <param name="b"></param>
+        /// <param name="dst"></param>
+        static abstract void Divide(in TA a, in TB b, ref TC dst);
+        /// <summary>
+        /// dst = a % b
+        /// </summary>
+        /// <param name="a"></param>
+        /// <param name="b"></param>
+        /// <param name="dst"></param>
+        static abstract void Module(in TA a, in TB b, ref TC dst);
+        /// <summary>
+        /// dst += a * b
+        /// </summary>
+        /// <param name="a"></param>
+        /// <param name="b"></param>
+        /// <param name="dst"></param>
+        static abstract TC MultiplyAndAdd(TA a, TB b, TC dst);
+    }
+}
diff --git a/NN/Abstraction/DType/IUnaryDTypeHandler.cs b/NN/Abstraction/DType/IUnaryDTypeHandler.cs
new file mode 100644
index 0000000..cfc24d5
--- /dev/null
+++ b/NN/Abstraction/DType/IUnaryDTypeHandler.cs
@@ -0,0 +1,18 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.DType
+{
+    public interface IUnaryDTypeHandler<T> where T: unmanaged
+    {
+        //static abstract void LeftShiftInplace(ref T src, int offset);
+        //static abstract void RightShiftAddInplace(ref T src, int offset);
+        //static abstract void Increase(ref T src);
+        //static abstract void Decrease(ref T src);
+        static abstract T Zero { get; }
+        static abstract T One { get; }
+    }
+}
diff --git a/NN/Abstraction/Data/INativeMemoryManager.cs b/NN/Abstraction/Data/INativeMemoryManager.cs
new file mode 100644
index 0000000..20fcd55
--- /dev/null
+++ b/NN/Abstraction/Data/INativeMemoryManager.cs
@@ -0,0 +1,13 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.Data
+{
+    public interface INativeMemoryManager
+    {
+        Memory<T> AllocateMemory<T>(int elems) where T: unmanaged;
+    }
+}
diff --git a/NN/Abstraction/Operators/IBinaryOperator.cs b/NN/Abstraction/Operators/IBinaryOperator.cs
new file mode 100644
index 0000000..2caf88b
--- /dev/null
+++ b/NN/Abstraction/Operators/IBinaryOperator.cs
@@ -0,0 +1,9 @@
+﻿using NN.Native.Basic;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public interface IBinaryOperator : IOperator
+    {
+
+    }
+}
diff --git a/NN/Abstraction/Operators/IMatmulOperator.cs b/NN/Abstraction/Operators/IMatmulOperator.cs
new file mode 100644
index 0000000..c2e36dc
--- /dev/null
+++ b/NN/Abstraction/Operators/IMatmulOperator.cs
@@ -0,0 +1,9 @@
+﻿using NN.Native.Basic;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public interface IMatmulOperator<TA, TB, TC>: ITernaryOperator where TA : unmanaged where TB : unmanaged where TC : unmanaged
+    {
+        static abstract void Exec(ReadOnlySpan<TA> a, ReadOnlySpan<TB> b, Span<TC> c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
+    }
+}
diff --git a/NN/Abstraction/Operators/IOperator.cs b/NN/Abstraction/Operators/IOperator.cs
new file mode 100644
index 0000000..354b1f0
--- /dev/null
+++ b/NN/Abstraction/Operators/IOperator.cs
@@ -0,0 +1,14 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public interface IOperator
+    {
+        static abstract bool IsThreadSafe { get; }
+        static abstract OperatorHandlerType HandlerType { get; }
+    }
+}
diff --git a/NN/Abstraction/Operators/ITernaryOperator.cs b/NN/Abstraction/Operators/ITernaryOperator.cs
new file mode 100644
index 0000000..ffbc3b3
--- /dev/null
+++ b/NN/Abstraction/Operators/ITernaryOperator.cs
@@ -0,0 +1,9 @@
+﻿using NN.Native.Basic;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public interface ITernaryOperator : IOperator
+    {
+
+    }
+}
diff --git a/NN/Abstraction/Operators/ITypeConvertOperator.cs b/NN/Abstraction/Operators/ITypeConvertOperator.cs
new file mode 100644
index 0000000..200f7f5
--- /dev/null
+++ b/NN/Abstraction/Operators/ITypeConvertOperator.cs
@@ -0,0 +1,11 @@
+﻿
+using NN.Native.Basic;
+using NN.Native.Operators.Common.Params;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public interface ITypeConvertOperator<TA, TB>: IBinaryOperator where TA: unmanaged where TB : unmanaged
+    {
+        static abstract void Exec(ReadOnlySpan<TA> a, Span<TB> b, in NativeLayout layoutA, in NativeLayout layoutB, in TypeConvertParam param);
+    }
+}
diff --git a/NN/Abstraction/Operators/IUnaryOperator.cs b/NN/Abstraction/Operators/IUnaryOperator.cs
new file mode 100644
index 0000000..8accbd2
--- /dev/null
+++ b/NN/Abstraction/Operators/IUnaryOperator.cs
@@ -0,0 +1,9 @@
+﻿using NN.Native.Basic;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public interface IUnaryOperator: IOperator
+    {
+
+    }
+}
diff --git a/NN/Abstraction/Operators/OperatorHandlerType.cs b/NN/Abstraction/Operators/OperatorHandlerType.cs
new file mode 100644
index 0000000..cf23436
--- /dev/null
+++ b/NN/Abstraction/Operators/OperatorHandlerType.cs
@@ -0,0 +1,16 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public enum OperatorHandlerType
+    {
+        Default = 0,
+        Naive,
+        X86,
+        MultiThreads
+    }
+}
diff --git a/NN/Basic/DType/BoolHandler.cs b/NN/Basic/DType/BoolHandler.cs
new file mode 100644
index 0000000..560af87
--- /dev/null
+++ b/NN/Basic/DType/BoolHandler.cs
@@ -0,0 +1,38 @@
+﻿//using NN.Native.Abstraction.DType;
+
+//namespace NN.Native.Basic.DType
+//{
+//    public class boolHandler : IUnaryDTypeHandler<bool>, IBinaryDTypeHandler<bool, bool>, ITernaryDTypeHandler<bool, bool, bool>, INativeConvertible<bool, bool>
+//    {
+//        #region unary
+//        public static bool Zero { get => false; }
+//        public static bool One { get => true; }
+//        #endregion
+
+//        #region binary
+//        public static void AddInplace(ref bool src, in bool addend) => throw new InvalidOperationException();
+//        public static void SubstractInplace(ref bool src, in bool subtrahend) => throw new InvalidOperationException();
+//        public static void MultiplyInplace(ref bool src, in bool multiplicator) => throw new InvalidOperationException();
+//        public static void DivideInplace(ref bool src, in bool divisor) => throw new InvalidOperationException();
+//        public static void ModuleInplace(ref bool src, in bool modulus) => throw new InvalidOperationException();
+//        //public static void LeftShift(ref bool src, ref bool dst, uint offset) => throw new InvalidOperationException();
+//        //public static void RightShift(ref bool src, ref bool dst, uint offset) => throw new InvalidOperationException();
+//        public static bool AreEqual(in bool a, in bool b) => a == b;
+//        public static bool IsGraeter(in bool a, in bool b) => throw new InvalidOperationException();
+//        public static bool IsLess(in bool a, in bool b) => throw new InvalidOperationException();
+//        #endregion
+
+//        #region ternary
+//        public static void Add(in bool a, in bool b, ref bool dst) => throw new InvalidOperationException();
+//        public static void Substract(in bool a, in bool b, ref bool dst) => throw new InvalidOperationException();
+//        public static void Multiply(in bool a, in bool b, ref bool dst) => throw new InvalidOperationException();
+//        public static void Divide(in bool a, in bool b, ref bool dst) => throw new InvalidOperationException();
+//        public static void Module(in bool a, in bool b, ref bool dst) => throw new InvalidOperationException();
+//        public static void MultiplyAndAdd(in bool a, in bool b, ref bool dst) => throw new InvalidOperationException();
+//        #endregion
+
+//        #region convert
+//        public static void Convert(in bool a, ref bool b) => b = a;
+//        #endregion
+//    }
+//}
diff --git a/NN/Basic/DType/FloatHandler.cs b/NN/Basic/DType/FloatHandler.cs
new file mode 100644
index 0000000..6fcde53
--- /dev/null
+++ b/NN/Basic/DType/FloatHandler.cs
@@ -0,0 +1,43 @@
+﻿using NN.Native.Abstraction.DType;
+using System.Runtime.CompilerServices;
+
+namespace NN.Native.Basic.DType
+{
+    public class FloatHandler: IUnaryDTypeHandler<float>, IBinaryDTypeHandler<float, float>, ITernaryDTypeHandler<float, float, float>, 
+            INativeConvertible<int, float>, INativeConvertible<bool, float>, INativeConvertible<float, float>
+    {
+        #region unary
+        public static float Zero { get => .0f; }
+        public static float One { get => 1.0f; }
+        #endregion
+
+        #region binary
+        public static void AddInplace(ref float src, in float addend) => src += addend;
+        public static void SubstractInplace(ref float src, in float subtrahend) => src -= subtrahend;
+        public static void MultiplyInplace(ref float src, in float multiplicator) => src *= multiplicator;
+        public static void DivideInplace(ref float src, in float divisor) => src /= divisor;
+        public static void ModuleInplace(ref float src, in float modulus) => src %= modulus;
+        //public static void LeftShift(ref float src, ref float dst, int offset) => throw new InvalidOperationException();
+        //public static void RightShift(ref float src, ref float dst, int offset) => throw new InvalidOperationException();
+        public static bool AreEqual(in float a, in float b) => a == b;
+        public static bool IsGraeter(in float a, in float b) => a > b;
+        public static bool IsLess(in float a, in float b) => a < b;
+        #endregion
+
+        #region ternary
+        public static void Add(in float a, in float b, ref float dst) => dst = a + b;
+        public static void Substract(in float a, in float b, ref float dst) => dst = a - b;
+        public static void Multiply(in float a, in float b, ref float dst) => dst = a * b;
+        public static void Divide(in float a, in float b, ref float dst) => dst = a / b;
+        public static void Module(in float a, in float b, ref float dst) => dst = a % b;
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static float MultiplyAndAdd(float a, float b, float dst) => dst + a * b;
+        #endregion
+
+        #region convert
+        public static void Convert(in float a, ref float b) => b = a;
+        public static void Convert(in int a, ref float b) => b = a;
+        public static void Convert(in bool a, ref float b) => b = a ? .1f: .0f;
+        #endregion
+    }
+}
diff --git a/NN/Basic/DType/Int32Handler.cs b/NN/Basic/DType/Int32Handler.cs
new file mode 100644
index 0000000..28b8744
--- /dev/null
+++ b/NN/Basic/DType/Int32Handler.cs
@@ -0,0 +1,39 @@
+﻿//using NN.Native.Abstraction.DType;
+
+//namespace NN.Native.Basic.DType
+//{
+//    public class Int32Handler : IUnaryDTypeHandler<int>, IBinaryDTypeHandler<int, int>, ITernaryDTypeHandler<int, int, int>, INativeConvertible<int, int>, INativeConvertible<bool, int>
+//    {
+//        #region unary
+//        public static int Zero { get => 0; }
+//        public static int One { get => 1; }
+//        #endregion
+
+//        #region binary
+//        public static void AddInplace(ref int src, in int addend) => src += addend;
+//        public static void SubstractInplace(ref int src, in int subtrahend) => src -= subtrahend;
+//        public static void MultiplyInplace(ref int src, in int multiplicator) => src *= multiplicator;
+//        public static void DivideInplace(ref int src, in int divisor) => src /= divisor;
+//        public static void ModuleInplace(ref int src, in int modulus) => src %= modulus;
+//        public static void LeftShift(in int src, ref int dst, int offset) => dst = src << offset;
+//        public static void RightShift(in int src, ref int dst, int offset) => dst = src >> offset;
+//        public static bool AreEqual(in int a, in int b) => a == b;
+//        public static bool IsGraeter(in int a, in int b) => a > b;
+//        public static bool IsLess(in int a, in int b) => a < b;
+//        #endregion
+
+//        #region ternary
+//        public static void Add(in int a, in int b, ref int dst) => dst = a + b;
+//        public static void Substract(in int a, in int b, ref int dst) => dst = a - b;
+//        public static void Multiply(in int a, in int b, ref int dst) => dst = a * b;
+//        public static void Divide(in int a, in int b, ref int dst) => dst = a / b;
+//        public static void Module(in int a, in int b, ref int dst) => dst = a % b;
+//        public static void MultiplyAndAdd(in int a, in int b, ref int dst) => dst += a * b;
+//        #endregion
+
+//        #region convert
+//        public static void Convert(in int a, ref int b) => b = a;
+//        public static void Convert(in bool a, ref int b) => b = a ? 1 : 0;
+//        #endregion
+//    }
+//}
diff --git a/NN/Basic/NativeLayout.cs b/NN/Basic/NativeLayout.cs
new file mode 100644
index 0000000..f2eb6fb
--- /dev/null
+++ b/NN/Basic/NativeLayout.cs
@@ -0,0 +1,287 @@
+﻿using NN.Native.Abstraction;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Basic
+{
+    public struct NativeLayout
+    {
+        // Modify the enumerator if changing it.
+        public readonly static int MAX_NDIM = 4;
+
+        internal int _ndim;
+        internal int[] _shape;
+        internal int[] _stride;
+
+        #region constructor
+        public NativeLayout()
+        {
+            _ndim = 0;
+            _shape = new int[MAX_NDIM];
+            _stride = new int[MAX_NDIM];
+        }
+        public NativeLayout(int[] shape)
+        {
+            _ndim = shape.Length;
+            Debug.Assert(_ndim > 0 && _ndim < MAX_NDIM);
+            _shape = new int[MAX_NDIM];
+            _stride = new int[MAX_NDIM];
+            shape.CopyTo(_shape, 0);
+            // init contiguous stride
+            int s = 1;
+            for(int i = 0; i < _ndim; i++)
+            {
+                _stride[_ndim - i - 1] = s;
+                s *= shape[_ndim - i - 1];
+            }
+        }
+        public NativeLayout(int[] shape, int[] stride)
+        {
+            _ndim = shape.Length;
+            Debug.Assert(_ndim > 0 && _ndim < MAX_NDIM);
+            _shape = new int[MAX_NDIM];
+            _stride = new int[MAX_NDIM];
+            shape.CopyTo(_shape, 0);
+            stride.CopyTo(_stride, 0);
+        }
+        public NativeLayout(ref NativeLayout layout)
+        {
+            _ndim = layout._ndim;
+            Debug.Assert(_ndim > 0 && _ndim < MAX_NDIM);
+            _shape = new int[MAX_NDIM];
+            _stride = new int[MAX_NDIM];
+            layout._shape.CopyTo(_shape, 0);
+            layout._stride.CopyTo(_stride, 0);
+        }
+        #endregion
+
+        #region readonly methods
+        public readonly bool IsScalar()
+        {
+            return _ndim == 1 && _shape[0] == 1;
+        }
+        public readonly bool IsEmpty()
+        {
+            if (_ndim == 0) return true;
+            for(int i = 0; i < _ndim; i++)
+            {
+                if (_shape[i] == 0) return true;
+            }
+            return false;
+        }
+        public readonly bool IsContiguous()
+        {
+            int expected = 1;
+            for(int i = 0; i < _ndim; i++)
+            {
+                if (_shape[_ndim - i - 1] != 1 && _stride[_ndim - i - 1] != expected) return false;
+                expected *= _shape[_ndim - i - 1];
+            }
+            // empty layout is supposed not to be contiguous.
+            return expected != 0;
+        }
+        public readonly int TotalElemCount()
+        {
+            int res = 1;
+            for (int i = 0; i < _ndim; i++) res *= _shape[i];
+            return res;
+        }
+        public readonly bool IsSameShape(ref NativeLayout other)
+        {
+            if (_ndim != other._ndim) return false;
+            return _shape.AsSpan().SequenceEqual(other._shape.AsSpan());
+        }
+        public readonly bool IsSameLayout(ref NativeLayout other)
+        {
+            return IsSameShape(ref other) && _stride.AsSpan().SequenceEqual(other._stride);
+        }
+        #endregion
+
+        public void RemoveAxisInplace(int axis)
+        {
+            Debug.Assert(_ndim >= 2 && axis < _ndim);
+            _ndim--;
+            for(int i = axis; i < _ndim; i++)
+            {
+                _shape[i] = _shape[i + 1];
+                _stride[i] = _stride[i + 1];
+            }
+        }
+
+        public void AddAxisInplace(int axis, int shape, int stride)
+        {
+            Debug.Assert(_ndim + 1 <= MAX_NDIM && axis <= _ndim && shape > 0);
+            _ndim++;
+            for(int i = _ndim - 1; i > axis; i--)
+            {
+                _shape[i] = _shape[i - 1];
+                _stride[i] = _stride[i - 1];
+            }
+            _shape[axis] = shape;
+            _stride[axis] = stride;
+        }
+
+        public void AddAxisInplace(int axis) => AddAxisInplace(axis, 1, _stride[axis] * _shape[axis]);
+
+        public override string ToString()
+        {
+            string res = "(";
+            if (_ndim == 0) res += "Scalar";
+            else
+            {
+                res += "shape = {";
+                for (int i = 0; i < _ndim; i++)
+                {
+                    res += _shape[i].ToString();
+                    if (i != _ndim - 1) res += ", ";
+                }
+                res += "}, stride = {";
+                for (int i = 0; i < _ndim; i++)
+                {
+                    res += _stride[i].ToString();
+                    if (i != _ndim - 1) res += ", ";
+                }
+                res += "}";
+            }
+            res += ")";
+            return res;
+        }
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Enumerator GetIndexEnumerator(in NativeLayout layout) => new Enumerator(layout);
+
+        public ref struct Enumerator
+        {
+            private int _index1, _index2, _index3;
+            private int _shape1, _shape2, _shape3;
+            private int _stride0, _stride1, _stride2, _stride3;
+            private int _product1, _product2, _product3;
+            private int _ndim;
+            private int _currentIndex;
+
+            [MethodImpl (MethodImplOptions.AggressiveInlining)]
+            public Enumerator(in NativeLayout layout)
+            {
+                _ndim = layout._ndim;
+                _currentIndex = 0;
+                if(_ndim == 4)
+                {
+                    _index1 = _index2 = _index3 = 0;
+                    _shape1 = layout._shape[1];
+                    _shape2 = layout._shape[2];
+                    _shape3 = layout._shape[3];
+                    _stride0 = layout._stride[0];
+                    _stride1 = layout._stride[1];
+                    _stride2 = layout._stride[2];
+                    _stride3 = layout._stride[3];
+                    _product1 = _stride1 * _shape1;
+                    _product2 = _stride2 * _shape2;
+                    _product3 = _stride3 * _shape3;
+                }
+                else if(_ndim == 3)
+                {
+                    _index1 = _index2 = 0;
+                    _shape1 = layout._shape[1];
+                    _shape2 = layout._shape[2];
+                    _stride0 = layout._stride[0];
+                    _stride1 = layout._stride[1];
+                    _stride2 = layout._stride[2];
+                    _product1 = _stride1 * _shape1;
+                    _product2 = _stride2 * _shape2;
+                }
+                else if (_ndim == 2)
+                {
+                    _index1 = 0;
+                    _shape1 = layout._shape[1];
+                    _stride0 = layout._stride[0];
+                    _stride1 = layout._stride[1];
+                    _product1 = _stride1 * _shape1;
+                }
+                else if (_ndim == 1)
+                {
+                    _stride0 = layout._stride[0];
+                }
+            }
+            /// <summary>
+            /// Move the index to next and return the old index value.
+            /// Note that there is no check for overflow. It should be used carefully and never be exposed to users.
+            /// </summary>
+            /// <returns></returns>
+            /// <exception cref="NotImplementedException"></exception>
+            public int MoveNext()
+            {
+                return _ndim switch
+                {
+                    1 => MoveWithNDim1(),
+                    2 => MoveWithNDim2(),
+                    3 => MoveWithNDim3(),
+                    4 => MoveWithNDim4(),
+                    _ => throw new NotImplementedException()
+                };
+            }
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            private int MoveWithNDim1()
+            {
+                var res = _currentIndex;
+                _currentIndex += _stride0;
+                return res;
+            }
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            private int MoveWithNDim2()
+            {
+                var res = _currentIndex;
+                _index1++;
+                _currentIndex += _stride1;
+                if(_index1 >= _shape1)
+                {
+                    _currentIndex += _stride0 - _product1;
+                }
+                return res;
+            }
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            private int MoveWithNDim3()
+            {
+                var res = _currentIndex;
+                _index2++;
+                _currentIndex += _stride2;
+                if (_index2 >= _shape2)
+                {
+                    _index1++;
+                    _currentIndex += _stride1 - _product2;
+                }
+                if (_index1 >= _shape1)
+                {
+                    _currentIndex += _stride0 - _product1;
+                }
+                return res;
+            }
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            private int MoveWithNDim4()
+            {
+                var res = _currentIndex;
+                _index3++;
+                _currentIndex += _stride3;
+                if (_index3 >= _shape3)
+                {
+                    _index2++;
+                    _currentIndex += _stride2 - _product3;
+                }
+                if (_index2 >= _shape2)
+                {
+                    _index1++;
+                    _currentIndex += _stride1 - _product2;
+                }
+                if (_index1 >= _shape1)
+                {
+                    _currentIndex += _stride0 - _product1;
+                }
+                return res;
+            }
+        }
+    }
+}
diff --git a/NN/Data/DefaultNativeMemoryManager.cs b/NN/Data/DefaultNativeMemoryManager.cs
new file mode 100644
index 0000000..27f5d5e
--- /dev/null
+++ b/NN/Data/DefaultNativeMemoryManager.cs
@@ -0,0 +1,20 @@
+﻿using NN.Native.Abstraction.Data;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading.Tasks;
+using System.Buffers;
+
+namespace NN.Native.Data
+{
+    public class DefaultNativeMemoryManager: INativeMemoryManager
+    {
+        public Memory<T> AllocateMemory<T>(int elems) where T: unmanaged
+        {
+            return new(new T[elems * Marshal.SizeOf<T>()]);
+        }
+    }
+}
diff --git a/NN/Data/NativeArray.cs b/NN/Data/NativeArray.cs
new file mode 100644
index 0000000..1982926
--- /dev/null
+++ b/NN/Data/NativeArray.cs
@@ -0,0 +1,74 @@
+﻿using NN.Native.Abstraction.Data;
+using NN.Native.Basic;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Data
+{
+    public struct NativeArray<T> where T: unmanaged
+    {
+        internal NativeStorage<T> _storage;
+        public NativeLayout _layout;
+        
+        public Span<T> Span { get => _storage._memory.Span; }
+
+        public NativeArray(NativeLayout layout, INativeMemoryManager memoryManager)
+        {
+            _layout = layout;
+            _storage = new NativeStorage<T>(memoryManager.AllocateMemory<T>(_layout.TotalElemCount()), 0);
+        }
+
+        public NativeArray(NativeLayout layout, Memory<T> memory)
+        {
+            _layout = layout;
+            _storage = new NativeStorage<T>(memory, 0);
+        }
+
+        public readonly bool IsMutable() => _storage._mutable;
+        public readonly bool IsMemoryOwner() => _storage._owned;
+    }
+
+    public partial class NativeArray
+    {
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static NativeArray<T> FromArray<T>(NativeLayout layout, T[] data) where T : unmanaged
+        {
+            return new NativeArray<T>(layout, new Memory<T>(data));
+        }
+        public static class Random
+        {
+            public static NativeArray<T> Normal<T>(NativeLayout layout, T minValue, T maxValue) where T: unmanaged
+            {
+                NativeArray<T> res = new NativeArray<T>(layout, new DefaultNativeMemoryManager());
+                if(typeof(T) == typeof(int))
+                {
+                    var span = MemoryMarshal.Cast<T, int>(res.Span);
+                    var random = new System.Random();
+                    for(int i = 0; i < span.Length; i++)
+                    {
+                        span[i] = random.Next((int)(object)minValue, (int)(object)maxValue);
+                    }
+                }
+                else if (typeof(T) == typeof(float))
+                {
+                    var span = MemoryMarshal.Cast<T, float>(res.Span);
+                    var random = new System.Random();
+                    for (int i = 0; i < span.Length; i++)
+                    {
+                        span[i] = random.NextSingle() * ((float)(object)maxValue - (float)(object)minValue) + (float)(object)minValue;
+                    }
+                }
+                else
+                {
+                    throw new NotImplementedException();
+                }
+                return res;
+            }
+        }
+    }
+}
diff --git a/NN/Data/NativeStorage.cs b/NN/Data/NativeStorage.cs
new file mode 100644
index 0000000..9c9decb
--- /dev/null
+++ b/NN/Data/NativeStorage.cs
@@ -0,0 +1,45 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Data
+{
+    internal struct NativeStorage<T> where T: unmanaged
+    {
+        internal Memory<T> _memory;
+        internal int _offset;
+        internal bool _mutable;
+        internal bool _owned;
+
+        public NativeStorage(Memory<T> memory, int offset, bool mutable = true, bool owned = true)
+        {
+            _memory = memory;
+            _offset = offset;
+            _mutable = mutable;
+            _owned = owned;
+        }
+
+
+        public override bool Equals([NotNullWhen(true)] object? obj)
+        {
+            if (obj is null or not NativeStorage<T>) return false;
+            else return this == (NativeStorage<T>)obj;
+        }
+        public override int GetHashCode()
+        {
+            return base.GetHashCode() + _offset;
+        }
+
+        public static bool operator ==(NativeStorage<T> a, NativeStorage<T> b)
+        {
+            return a._memory.Equals(b._memory) && a._offset == b._offset;
+        }
+        public static bool operator !=(NativeStorage<T> a, NativeStorage<T> b)
+        {
+            return !(a == b);
+        }
+    }
+}
diff --git a/NN/Extensions/IndexOffsetConvertExtension.cs b/NN/Extensions/IndexOffsetConvertExtension.cs
new file mode 100644
index 0000000..d4f5120
--- /dev/null
+++ b/NN/Extensions/IndexOffsetConvertExtension.cs
@@ -0,0 +1,34 @@
+﻿using NN.Native.Basic;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Extensions
+{
+    internal static class IndexOffsetConvertExtension
+    {
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public unsafe static int IndicesToOffset(this NativeLayout layout, int* indices)
+        {
+            int res = 0;
+            for (int i = 0; i < layout._ndim; i++)
+            {
+                res += indices[i] * layout._stride[i];
+            }
+            return res;
+        }
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public unsafe static void OffsetToIndices(this NativeLayout layout, int* indices, int offset)
+        {
+            for (int i = 0; i < layout._ndim; i++)
+            {
+                int idx = offset / layout._stride[i];
+                indices[i] = idx;
+                offset -= idx * layout._stride[i];
+            }
+        }
+    }
+}
diff --git a/NN/Extensions/PrintArrayExtension.cs b/NN/Extensions/PrintArrayExtension.cs
new file mode 100644
index 0000000..d607245
--- /dev/null
+++ b/NN/Extensions/PrintArrayExtension.cs
@@ -0,0 +1,68 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Basic;
+using NN.Native.Data;
+
+namespace NN.Native.Extensions
+{
+    public static class PrintArrayExtension
+    {
+        public static string Print<T>(this NativeArray<T> array) where T: unmanaged
+        {
+            var layout = array._layout;
+            var indexEnumerator = NativeLayout.GetIndexEnumerator(layout);
+
+            StringBuilder r = new StringBuilder($"NativeArray {layout}:\n");
+            var span = array.Span;
+            for (int i = 0; i < layout.TotalElemCount(); i++)
+            {
+                int mod = 1;
+                for (int j = layout._ndim - 1; j >= 0; j--)
+                {
+                    mod *= layout._shape[j];
+                    if (i % mod == 0)
+                    {
+                        r.Append("[");
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+                r.Append(" ").Append(span[indexEnumerator.MoveNext()]);
+
+                if ((i + 1) % layout._shape[layout._ndim - 1] != 0) r.Append(",");
+
+                r.Append(" ");
+                mod = 1;
+                int hit_times = 0;
+                for (int j = layout._ndim - 1; j >= 0; j--)
+                {
+                    mod *= layout._shape[j];
+                    if ((i + 1) % mod == 0)
+                    {
+                        r.Append("]");
+                        hit_times++;
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+                if (hit_times > 0 && hit_times < layout._ndim)
+                {
+                    r.Append(",\n");
+                    for (int j = 0; j < layout._ndim - hit_times; j++)
+                    {
+                        r.Append(" ");
+                    }
+                }
+            }
+            // r.Append("\n");
+            return r.ToString();
+        }
+    }
+}
diff --git a/NN/NN.Native.csproj b/NN/NN.Native.csproj
new file mode 100644
index 0000000..eb9482b
--- /dev/null
+++ b/NN/NN.Native.csproj
@@ -0,0 +1,15 @@
+﻿<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
+    <LangVersion>preview</LangVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Folder Include="Operators\X86\" />
+  </ItemGroup>
+
+</Project>
diff --git a/NN/Operators/Common/Params/MatmulParam.cs b/NN/Operators/Common/Params/MatmulParam.cs
new file mode 100644
index 0000000..8134896
--- /dev/null
+++ b/NN/Operators/Common/Params/MatmulParam.cs
@@ -0,0 +1,14 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Operators.Common.Params
+{
+    public struct MatmulParam
+    {
+        public bool TransposeA;
+        public bool TransposeB;
+    }
+}
diff --git a/NN/Operators/Common/Params/TypeConvertParam.cs b/NN/Operators/Common/Params/TypeConvertParam.cs
new file mode 100644
index 0000000..09aef48
--- /dev/null
+++ b/NN/Operators/Common/Params/TypeConvertParam.cs
@@ -0,0 +1,15 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Operators.Common.Params
+{
+    public struct TypeConvertParam
+    {
+        public bool Transpose;
+        public int DimA;
+        public int DimB;
+    }
+}
diff --git a/NN/Operators/Common/TypeConvertOperatorBase.cs b/NN/Operators/Common/TypeConvertOperatorBase.cs
new file mode 100644
index 0000000..1195f6b
--- /dev/null
+++ b/NN/Operators/Common/TypeConvertOperatorBase.cs
@@ -0,0 +1,19 @@
+﻿using NN.Native.Abstraction.Operators;
+using NN.Native.Basic;
+using NN.Native.Operators.Common.Params;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Operators.Common
+{
+    public abstract class TypeConvertOperatorBase
+    {
+        public static void DeduceLayout(in NativeLayout src, in TypeConvertParam param, out NativeLayout dst)
+        {
+            dst = new(src._shape);
+        }
+    }
+}
diff --git a/NN/Operators/Naive/MatmulOperator.cs b/NN/Operators/Naive/MatmulOperator.cs
new file mode 100644
index 0000000..fdbdf4b
--- /dev/null
+++ b/NN/Operators/Naive/MatmulOperator.cs
@@ -0,0 +1,35 @@
+﻿using NN.Native.Abstraction.DType;
+using NN.Native.Basic;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Operators.Naive
+{
+    public class MatmulOperator<TA, TB, TC, THandler> where TA: unmanaged where TB: unmanaged where TC: unmanaged where THandler: ITernaryDTypeHandler<TA, TB, TC>, IUnaryDTypeHandler<TC>
+    {
+        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        public static unsafe void Exec(ReadOnlySpan<TA> a, ReadOnlySpan<TB> b, Span<TC> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        {
+            // The array should be contiguous here
+            int aRows = layoutA._shape[0];
+            int aCols = layoutA._shape[1];
+            int bCols = layoutB._shape[1];
+            for(int i = 0; i < aRows; i++)
+            {
+                for(int j = 0; j < bCols; j++)
+                {
+                    var res = THandler.Zero;
+                    for(int k = 0; k < aCols; k++)
+                    {
+                        res = THandler.MultiplyAndAdd(a[i * layoutA._shape[1] + k], b[k * layoutB._shape[1] + j], res);
+                    }
+                    c[i * layoutC._shape[1] + j] = res;
+                }
+            }
+        }
+    }
+}
diff --git a/NN/Operators/Naive/TypeConvertOperator.cs b/NN/Operators/Naive/TypeConvertOperator.cs
new file mode 100644
index 0000000..615cd39
--- /dev/null
+++ b/NN/Operators/Naive/TypeConvertOperator.cs
@@ -0,0 +1,40 @@
+﻿using NN.Native.Abstraction.DType;
+using NN.Native.Abstraction.Operators;
+using NN.Native.Basic;
+using NN.Native.Extensions;
+using NN.Native.Operators.Common;
+using NN.Native.Operators.Common.Params;
+
+namespace NN.Native.Operators.Naive
+{
+    public class TypeConvertOperator<TA, TB, THandler>: TypeConvertOperatorBase where TA: unmanaged where TB: unmanaged where THandler: INativeConvertible<TA, TB>
+    {
+        public static bool IsThreadSafe { get => false; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+        public static unsafe void Exec(ReadOnlySpan<TA> a, Span<TB> b, in NativeLayout layoutA, in NativeLayout layoutB, in TypeConvertParam param)
+        {
+            int totalElems = layoutA.TotalElemCount();
+            if (!param.Transpose)
+            {
+                var indexEnumeratorA = NativeLayout.GetIndexEnumerator(layoutA);
+                var indexEnumeratorB = NativeLayout.GetIndexEnumerator(layoutB);
+                for (int i = 0; i < totalElems; i++)
+                {
+                    THandler.Convert(a[indexEnumeratorA.MoveNext()], ref b[indexEnumeratorB.MoveNext()]);
+                }
+            }
+            else
+            {
+                int* indices = stackalloc int[layoutB._ndim];
+                for(int offset = 0; offset < totalElems; offset++)
+                {
+                    layoutA.OffsetToIndices(indices, offset);
+                    indices[param.DimA] ^= indices[param.DimB];
+                    indices[param.DimB] ^= indices[param.DimA];
+                    indices[param.DimA] ^= indices[param.DimB];
+                    THandler.Convert(a[offset], ref b[layoutB.IndicesToOffset(indices)]);
+                }
+            }
+        }
+    }
+}

From 2fdedf3922376ddfd071d73b262a0f65a46e8a6e Mon Sep 17 00:00:00 2001
From: AsakusaRinne <AsakusaRinne@gmail.com>
Date: Wed, 2 Nov 2022 00:01:55 +0800
Subject: [PATCH 2/5] switch to single generic implementation.

---
 NN.BenchmarkTest/Helper/DimensionHelper.cs    |  30 +++++
 NN.BenchmarkTest/MatmulBenchmark.cs           | 109 +++++++++---------
 NN.BenchmarkTest/NN.BenchmarkTest.csproj      |   3 +-
 NN.BenchmarkTest/Program.cs                   |  13 ++-
 .../comparison/MatmulBenchmark.cs             |  69 +++++++++++
 NN.UnitTests/NN.UnitTests.csproj              |   4 +
 .../{ => Operators/Naive}/NaiveTest.cs        |  14 +--
 NN.UnitTests/Operators/X86/MatmulTest.cs      |  43 +++++++
 NN/Abstraction/DType/IBinaryDTypeHandler.cs   |  15 +--
 NN/Abstraction/DType/INativeConvertible.cs    |   6 +-
 NN/Abstraction/DType/INativeDType.cs          |  15 ---
 NN/Abstraction/DType/INativeDTypeHandler.cs   |  14 +++
 NN/Abstraction/DType/ITernaryDTypeHandler.cs  |  33 +++---
 NN/Abstraction/DType/IUnaryDTypeHandler.cs    |   8 +-
 NN/Abstraction/Operators/IMatmulOperator.cs   |   8 +-
 NN/Abstraction/Operators/IOperator.cs         |   5 +
 .../Operators/ITypeConvertOperator.cs         |   4 +
 NN/Basic/DType/FloatHandler.cs                |  44 ++++---
 NN/Data/DefaultNativeMemoryManager.cs         |   2 +-
 NN/Data/NativeArray.cs                        |   2 +-
 NN/Data/NativeStorage.cs                      |   2 +-
 NN/NN.Native.csproj                           |   4 +-
 NN/Operators/Naive/MatmulOperator.cs          |  42 +++++--
 NN/Operators/Naive/TypeConvertOperator.cs     |  20 +++-
 NN/Operators/X86/MatmulOperator.cs            |  66 +++++++++++
 25 files changed, 415 insertions(+), 160 deletions(-)
 create mode 100644 NN.BenchmarkTest/Helper/DimensionHelper.cs
 create mode 100644 NN.BenchmarkTest/comparison/MatmulBenchmark.cs
 rename NN.UnitTests/{ => Operators/Naive}/NaiveTest.cs (66%)
 create mode 100644 NN.UnitTests/Operators/X86/MatmulTest.cs
 delete mode 100644 NN/Abstraction/DType/INativeDType.cs
 create mode 100644 NN/Abstraction/DType/INativeDTypeHandler.cs
 create mode 100644 NN/Operators/X86/MatmulOperator.cs

diff --git a/NN.BenchmarkTest/Helper/DimensionHelper.cs b/NN.BenchmarkTest/Helper/DimensionHelper.cs
new file mode 100644
index 0000000..691d24f
--- /dev/null
+++ b/NN.BenchmarkTest/Helper/DimensionHelper.cs
@@ -0,0 +1,30 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.BenchmarkTest.Helper
+{
+    public class DimensionHelper
+    {
+        private int[][] _dimensions;
+        public DimensionHelper(params int[][] dimensions)
+        {
+            _dimensions = dimensions;
+        }
+        public int[] this[int idx]
+        {
+            get => _dimensions[idx];
+        }
+        public override string ToString()
+        {
+            string res = "";
+            foreach (var dim in _dimensions)
+            {
+                res += $"({string.Join(',', dim)}) ";
+            }
+            return res;
+        }
+    }
+}
diff --git a/NN.BenchmarkTest/MatmulBenchmark.cs b/NN.BenchmarkTest/MatmulBenchmark.cs
index 1fe773d..e0bcf23 100644
--- a/NN.BenchmarkTest/MatmulBenchmark.cs
+++ b/NN.BenchmarkTest/MatmulBenchmark.cs
@@ -1,64 +1,59 @@
-﻿using NN.Native.Operators;
-using BenchmarkDotNet.Attributes;
-using System.Reflection;
-using NN.Native.Data;
+﻿using BenchmarkDotNet.Attributes;
+using NN.Native.Basic.DType;
 using NN.Native.Basic;
-using static Tensorflow.Binding;
+using NN.Native.Data;
 using NN.Native.Operators.Naive;
-using NN.Native.Basic.DType;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Jobs;
+using NN.BenchmarkTest.Helper;
 
-[MemoryDiagnoser]
-[BenchmarkCategory("MatmulFloat32")]
-public class MatmulFloat32Benchmark
+namespace NN.BenchmarkTest
 {
-    public IEnumerable<int[][]> DimensionValues => new int[][][] {
-            new int[][]{ new int[] { 3, 4 }, new int[] { 4, 5} },
-            new int[][]{ new int[] { 32, 64 }, new int[] { 64, 48} },
-            new int[][]{ new int[] { 256, 512 }, new int[] { 512, 192} },
-            new int[][]{ new int[] { 8, 1024 }, new int[] { 1024, 1} },
-        };
-    [ParamsSource(nameof(DimensionValues))]
-    public int[][] Dimensions { get; set; }
-
-    NativeArray<float> ArrayLeft { get; set; }
-    NativeArray<float> ArrayRight { get; set; }
-    NumSharp.NDArray NDArrayLeft { get; set; }
-    NumSharp.NDArray NDArrayRight { get; set; }
-    Tensorflow.NumPy.NDArray TFNetTensorLeft { get; set; }
-    Tensorflow.NumPy.NDArray TFNetTensorRight { get; set; }
-    [GlobalSetup(Targets = new[] { nameof(TensornetMatmul) })]
-    public void TensornetSetup()
-    {
-        ArrayLeft = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[0]), -100, 100);
-        ArrayRight = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[1]), -100, 100);
-    }
-    [GlobalSetup(Targets = new[] { nameof(NumSharpMatmul) })]
-    public void NumSharpSetup()
+    [MemoryDiagnoser] 
+    [BenchmarkCategory("Matmul")]
+    //[SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.NetCoreApp31, baseline: true)]
+    //[SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.Net60)]
+    [SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.Net70)]
+    public class MatmulFloat32Benchmark
     {
-        NDArrayLeft = NumSharp.np.random.uniform(-100, 100, new NumSharp.Shape(Dimensions[0]));
-        NDArrayRight = NumSharp.np.random.uniform(-100, 100, new NumSharp.Shape(Dimensions[1]));
-    }
-    [GlobalSetup(Targets = new[] { nameof(TFNetMatmul) })]
-    public void TFNetSetup()
-    {
-        TFNetTensorLeft = Tensorflow.NumPy.np.random.normal(0, 100, new Tensorflow.Shape(Dimensions[0]));
-        TFNetTensorRight = Tensorflow.NumPy.np.random.normal(0, 100, new Tensorflow.Shape(Dimensions[1]));
-    }
+        public IEnumerable<DimensionHelper> DimensionValues => new DimensionHelper[] {
+            new DimensionHelper( new int[] { 6, 8 }, new int[] { 8, 11} ),
+            new DimensionHelper( new int[] { 32, 64 }, new int[] { 64, 96} ),
+            new DimensionHelper(new int[] { 256, 512 }, new int[] { 512, 768} ),
+            new DimensionHelper(new int[] { 1000, 1000 }, new int[] { 1000, 1000} ),
+            new DimensionHelper(new int[] { 2, 1024 }, new int[] { 1024, 1} ),
+            new DimensionHelper(new int[] { 1024, 1 }, new int[] { 2, 1024} )
+        };
+        [ParamsSource(nameof(DimensionValues))]
+        public DimensionHelper Dimensions { get; set; }
 
-    [Benchmark]
-    public void TensornetMatmul()
-    {
-        NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
-        MatmulOperator<float, float, float, FloatHandler>.Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
-    }
-    [Benchmark]
-    public void NumSharpMatmul()
-    {
-        NumSharp.np.matmul(NDArrayLeft, NDArrayRight);
-    }
-    [Benchmark]
-    public void TFNetMatmul()
-    {
-        tf.matmul(TFNetTensorLeft, TFNetTensorRight);
+        NativeArray<float> ArrayLeft { get; set; }
+        NativeArray<float> ArrayRight { get; set; }
+        [GlobalSetup(Targets = new[] { nameof(X86Matmul), nameof(NaiveMatmul) })]
+        public void GlobalSetup()
+        {
+            ArrayLeft = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[0]), -100, 100);
+            ArrayRight = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[1]), -100, 100);
+        }
+        [Benchmark]
+        public void NaiveMatmul()
+        {
+            NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
+#if NET7_0_OR_GREATER
+            MatmulOperator<float>.Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+#else
+            new MatmulOperator<float, Float32Handler>().Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+#endif
+        }
+        [Benchmark]
+        public void X86Matmul()
+        {
+            NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
+#if NET7_0_OR_GREATER
+            NN.Native.Operators.X86.MatmulOperator<float>.Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+#else
+            new NN.Native.Operators.X86.MatmulOperator<float, Float32Handler>().Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+#endif
+        }
     }
-}
\ No newline at end of file
+}
diff --git a/NN.BenchmarkTest/NN.BenchmarkTest.csproj b/NN.BenchmarkTest/NN.BenchmarkTest.csproj
index 1bd6885..8fc8137 100644
--- a/NN.BenchmarkTest/NN.BenchmarkTest.csproj
+++ b/NN.BenchmarkTest/NN.BenchmarkTest.csproj
@@ -2,7 +2,8 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFrameworks>net7.0;net6.0;netcoreapp3.1</TargetFrameworks>
+    <LangVersion>preview</LangVersion>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
diff --git a/NN.BenchmarkTest/Program.cs b/NN.BenchmarkTest/Program.cs
index c81bdf8..0a7d0eb 100644
--- a/NN.BenchmarkTest/Program.cs
+++ b/NN.BenchmarkTest/Program.cs
@@ -1,4 +1,13 @@
 ﻿using BenchmarkDotNet.Running;
 
-var summary = BenchmarkRunner.Run(typeof(Program).Assembly);
-Console.WriteLine(summary);
\ No newline at end of file
+namespace NN.BenchmarkTest
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            var summary = BenchmarkRunner.Run(typeof(Program).Assembly);
+            Console.WriteLine(summary);
+        }
+    }
+}
diff --git a/NN.BenchmarkTest/comparison/MatmulBenchmark.cs b/NN.BenchmarkTest/comparison/MatmulBenchmark.cs
new file mode 100644
index 0000000..41829c0
--- /dev/null
+++ b/NN.BenchmarkTest/comparison/MatmulBenchmark.cs
@@ -0,0 +1,69 @@
+﻿using NN.Native.Operators;
+using BenchmarkDotNet.Attributes;
+using System.Reflection;
+using NN.Native.Data;
+using NN.Native.Basic;
+using static Tensorflow.Binding;
+using NN.Native.Operators.Naive;
+using NN.Native.Basic.DType;
+
+[MemoryDiagnoser]
+[BenchmarkCategory("Comparison-MatmulFloat32")]
+internal class MatmulFloat32Benchmark
+{
+    public IEnumerable<int[][]> DimensionValues => new int[][][] {
+            new int[][]{ new int[] { 3, 4 }, new int[] { 4, 5} },
+            new int[][]{ new int[] { 32, 64 }, new int[] { 64, 48} },
+            new int[][]{ new int[] { 256, 512 }, new int[] { 512, 192} },
+            new int[][]{ new int[] { 8, 1024 }, new int[] { 1024, 1} },
+        };
+    [ParamsSource(nameof(DimensionValues))]
+    public int[][] Dimensions { get; set; }
+
+    NativeArray<float> ArrayLeft { get; set; }
+    NativeArray<float> ArrayRight { get; set; }
+    NumSharp.NDArray NDArrayLeft { get; set; }
+    NumSharp.NDArray NDArrayRight { get; set; }
+    Tensorflow.NumPy.NDArray TFNetTensorLeft { get; set; }
+    Tensorflow.NumPy.NDArray TFNetTensorRight { get; set; }
+    [GlobalSetup(Targets = new[] { nameof(GenericMathMatmul) })]
+    public void TensornetSetup()
+    {
+        ArrayLeft = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[0]), -100, 100);
+        ArrayRight = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[1]), -100, 100);
+    }
+    [GlobalSetup(Targets = new[] { nameof(NumSharpMatmul) })]
+    public void NumSharpSetup()
+    {
+        NDArrayLeft = NumSharp.np.random.uniform(-100, 100, new NumSharp.Shape(Dimensions[0]));
+        NDArrayRight = NumSharp.np.random.uniform(-100, 100, new NumSharp.Shape(Dimensions[1]));
+    }
+    [GlobalSetup(Targets = new[] { nameof(TFNetMatmul) })]
+    public void TFNetSetup()
+    {
+        TFNetTensorLeft = Tensorflow.NumPy.np.random.normal(0, 100, new Tensorflow.Shape(Dimensions[0]));
+        TFNetTensorRight = Tensorflow.NumPy.np.random.normal(0, 100, new Tensorflow.Shape(Dimensions[1]));
+    }
+
+    [Benchmark]
+    public void GenericMathMatmul()
+    {
+        //NumSharp.np.matmul(NDArrayLeft, NDArrayRight);
+        NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
+#if NET7_0_OR_GREATER
+        MatmulOperator<float>.Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+#else
+        new MatmulOperator<float, Float32Handler>().Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+#endif
+    }
+    [Benchmark]
+    public void NumSharpMatmul()
+    {
+        NumSharp.np.matmul(NDArrayLeft, NDArrayRight);
+    }
+    [Benchmark]
+    public void TFNetMatmul()
+    {
+        tf.matmul(TFNetTensorLeft, TFNetTensorRight);
+    }
+}
\ No newline at end of file
diff --git a/NN.UnitTests/NN.UnitTests.csproj b/NN.UnitTests/NN.UnitTests.csproj
index 9631cd9..53a6065 100644
--- a/NN.UnitTests/NN.UnitTests.csproj
+++ b/NN.UnitTests/NN.UnitTests.csproj
@@ -25,4 +25,8 @@
     <ProjectReference Include="..\NN\NN.Native.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Helper\" />
+  </ItemGroup>
+
 </Project>
diff --git a/NN.UnitTests/NaiveTest.cs b/NN.UnitTests/Operators/Naive/NaiveTest.cs
similarity index 66%
rename from NN.UnitTests/NaiveTest.cs
rename to NN.UnitTests/Operators/Naive/NaiveTest.cs
index 6cbf209..b40e08e 100644
--- a/NN.UnitTests/NaiveTest.cs
+++ b/NN.UnitTests/Operators/Naive/NaiveTest.cs
@@ -6,7 +6,7 @@
 using NN.Native.Operators.Common.Params;
 using Xunit.Abstractions;
 
-namespace NN.UnitTests
+namespace NN.UnitTests.Naive
 {
     public class NaiveTest
     {
@@ -14,7 +14,7 @@ public class NaiveTest
 
         public NaiveTest(ITestOutputHelper output)
         {
-            this._output = output;
+            _output = output;
         }
         [Fact]
         public void TypeConvertTest()
@@ -22,7 +22,7 @@ public void TypeConvertTest()
             var array = NativeArray.FromArray(new NativeLayout(new int[] { 3, 4 }), new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 });
             _output.WriteLine(array.Print());
             var convertedArray = new NativeArray<float>(new NativeLayout(new int[] { 4, 3 }), new DefaultNativeMemoryManager());
-            TypeConvertOperator<int, float, FloatHandler>.Exec(array.Span, convertedArray.Span, array._layout, convertedArray._layout, new TypeConvertParam()
+            TypeConvertOperator<int, float, Float32Handler>.Exec(array.Span, convertedArray.Span, array._layout, convertedArray._layout, new TypeConvertParam()
             {
                 Transpose = true,
                 DimA = 0,
@@ -33,10 +33,10 @@ public void TypeConvertTest()
         [Fact]
         public void MatmulTest()
         {
-            var a = NativeArray.FromArray(new NativeLayout(new int[] { 3, 4 }), new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 });
-            var b = NativeArray.FromArray(new NativeLayout(new int[] { 4, 2 }), new int[] { 1, 2, 1, 2, 1, 1, 2, 3 });
-            var c = new NativeArray<int>(new NativeLayout(new int[] { 3, 2 }), new DefaultNativeMemoryManager());
-            MatmulOperator<int, int, int, Int32Handler>.Exec(a.Span, b.Span, c.Span, a._layout, b._layout, c._layout);
+            var a = NativeArray.FromArray(new NativeLayout(new int[] { 3, 4 }), new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 });
+            var b = NativeArray.FromArray(new NativeLayout(new int[] { 4, 2 }), new float[] { 1, 2, 1, 2, 1, 1, 2, 3 });
+            var c = new NativeArray<float>(new NativeLayout(new int[] { 3, 2 }), new DefaultNativeMemoryManager());
+            Native.Operators.X86.MatmulOperator<float>.Exec(a.Span, b.Span, c.Span, a._layout, b._layout, c._layout);
             _output.WriteLine(c.Print());
         }
     }
diff --git a/NN.UnitTests/Operators/X86/MatmulTest.cs b/NN.UnitTests/Operators/X86/MatmulTest.cs
new file mode 100644
index 0000000..eeeb54f
--- /dev/null
+++ b/NN.UnitTests/Operators/X86/MatmulTest.cs
@@ -0,0 +1,43 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Data;
+using NN.Native.Basic;
+
+namespace NN.UnitTests.Operators.X86
+{
+    public class MatmulTest
+    {
+        public (int[], int[])[] Shapes { get; set; } = new(int[], int[])[] {
+                (new int[] {2, 4}, new int[] {4, 4}),
+                (new int[] {3, 7}, new int[] {7, 16}),
+                (new int[] {1, 20}, new int[] {20, 1}),
+                (new int[] {10, 1}, new int[] {1, 20}),
+                (new int[] {100, 100}, new int[] {100, 101}),
+            };
+
+        [Fact]
+        public void Float32Test()
+        {
+            foreach(var shape in Shapes)
+            {
+                var a = NativeArray.Random.Normal<float>(new NativeLayout(shape.Item1), -1000, 1000);
+                var b = NativeArray.Random.Normal<float>(new NativeLayout(shape.Item2), -1000, 1000);
+                var naiveResult = new NativeArray<float>(new NativeLayout(new int[] { shape.Item1[0], shape.Item2[1] }), new DefaultNativeMemoryManager());
+                var x86Result = new NativeArray<float>(new NativeLayout(new int[] { shape.Item1[0], shape.Item2[1] }), new DefaultNativeMemoryManager());
+                Native.Operators.Naive.MatmulOperator<float>.Exec(a.Span, b.Span, naiveResult.Span, a._layout, b._layout, naiveResult._layout);
+                Native.Operators.X86.MatmulOperator<float>.Exec(a.Span, b.Span, x86Result.Span, a._layout, b._layout, x86Result._layout);
+                var enumeratorNaive = NativeLayout.GetIndexEnumerator(naiveResult._layout);
+                var enumeratorX86 = NativeLayout.GetIndexEnumerator(x86Result._layout);
+                var naiveSpan = naiveResult.Span;
+                var x86SPan = x86Result.Span;
+                for(int i = 0; i < naiveResult._layout.TotalElemCount(); i++)
+                {
+                    Assert.Equal(naiveSpan[enumeratorNaive.MoveNext()], x86SPan[enumeratorX86.MoveNext()]);
+                }
+            }
+        }
+    }
+}
diff --git a/NN/Abstraction/DType/IBinaryDTypeHandler.cs b/NN/Abstraction/DType/IBinaryDTypeHandler.cs
index 8d4f2d4..a6cb9c6 100644
--- a/NN/Abstraction/DType/IBinaryDTypeHandler.cs
+++ b/NN/Abstraction/DType/IBinaryDTypeHandler.cs
@@ -6,29 +6,22 @@
 
 namespace NN.Native.Abstraction.DType
 {
-    public unsafe interface IBinaryDTypeHandler<TA, TB> where TA : unmanaged where TB : unmanaged
+    public unsafe interface IBinaryDTypeHandler<T> where T : unmanaged
     {
-        static abstract void AddInplace(ref TA src, in TB addend);
-        static abstract void SubstractInplace(ref TA src, in TB subtrahend);
-        static abstract void MultiplyInplace(ref TA src, in TB multiplicator);
-        static abstract void DivideInplace(ref TA src, in TB divisor);
-        static abstract void ModuleInplace(ref TA src, in TB modulus);
-        //static abstract void LeftShift(ref TA src, ref TB dst, int offset);
-        //static abstract void RightShift(ref TA src, ref TB dst, int offset);
-        static abstract bool AreEqual (in TA a, in TB b);
+        bool AreEqual (T a, T b);
         /// <summary>
         /// Return if a is greater than b.
         /// </summary>
         /// <param name="a"></param>
         /// <param name="b"></param>
         /// <returns></returns>
-        static abstract bool IsGraeter(in TA a, in TB b);
+        bool IsGraeter(T a, T b);
         /// <summary>
         /// Return if a is less than b.
         /// </summary>
         /// <param name="a"></param>
         /// <param name="b"></param>
         /// <returns></returns>
-        static abstract bool IsLess(in TA a, in TB b);
+        bool IsLess(T a, T b);
     }
 }
diff --git a/NN/Abstraction/DType/INativeConvertible.cs b/NN/Abstraction/DType/INativeConvertible.cs
index 65c84a4..7f1badf 100644
--- a/NN/Abstraction/DType/INativeConvertible.cs
+++ b/NN/Abstraction/DType/INativeConvertible.cs
@@ -8,6 +8,10 @@ namespace NN.Native.Abstraction.DType
 {
     public interface INativeConvertible<TA, TB> where TA: unmanaged where TB: unmanaged
     {
-        static abstract void Convert(in TA a, ref TB b);
+#if NET7_0_OR_GREATER
+        static abstract TB Convert(TA a);
+#else
+        TB Convert(TA a);
+#endif
     }
 }
diff --git a/NN/Abstraction/DType/INativeDType.cs b/NN/Abstraction/DType/INativeDType.cs
deleted file mode 100644
index eb9a2be..0000000
--- a/NN/Abstraction/DType/INativeDType.cs
+++ /dev/null
@@ -1,15 +0,0 @@
-﻿using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace NN.Native.Abstraction.DType
-{
-    //internal interface INativeDTypeHandler<TA, TB, TC>: 
-    //    IBinaryDTypeHandler<TA, TC>, IBinaryDTypeHandler<TB, TC>, ITernaryDTypeHandler<TA, TB, TC>
-    //    where TA: unmanaged where TB: unmanaged where TC: unmanaged
-    //{
-        
-    //}
-}
diff --git a/NN/Abstraction/DType/INativeDTypeHandler.cs b/NN/Abstraction/DType/INativeDTypeHandler.cs
new file mode 100644
index 0000000..4901a6e
--- /dev/null
+++ b/NN/Abstraction/DType/INativeDTypeHandler.cs
@@ -0,0 +1,14 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.DType
+{
+    public interface INativeDTypeHandler<T> :
+        IBinaryDTypeHandler<T>, IUnaryDTypeHandler<T>, ITernaryDTypeHandler<T> where T : unmanaged
+    {
+
+    }
+}
diff --git a/NN/Abstraction/DType/ITernaryDTypeHandler.cs b/NN/Abstraction/DType/ITernaryDTypeHandler.cs
index 0bbf4f9..488e407 100644
--- a/NN/Abstraction/DType/ITernaryDTypeHandler.cs
+++ b/NN/Abstraction/DType/ITernaryDTypeHandler.cs
@@ -1,54 +1,51 @@
 ﻿using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading.Tasks;
 
 namespace NN.Native.Abstraction.DType
 {
-    public interface ITernaryDTypeHandler<TA, TB, TC> where TA : unmanaged where TB : unmanaged where TC: unmanaged
+    public interface ITernaryDTypeHandler<T> where T : unmanaged
     {
         /// <summary>
-        /// dst = a + b
+        /// a + b
         /// </summary>
         /// <param name="a"></param>
         /// <param name="b"></param>
-        /// <param name="dst"></param>
-        static abstract void Add(in TA a, in TB b, ref TC dst);
+        T Add(T a, T b);
         /// <summary>
-        /// dst = a - b
+        /// a - b
         /// </summary>
         /// <param name="a"></param>
         /// <param name="b"></param>
-        /// <param name="dst"></param>
-        static abstract void Substract(in TA a, in TB b, ref TC dst);
+        T Substract(T a, T b);
         /// <summary>
-        /// dst = a * b
+        /// a * b
         /// </summary>
         /// <param name="a"></param>
         /// <param name="b"></param>
-        /// <param name="dst"></param>
-        static abstract void Multiply(in TA a, in TB b, ref TC dst);
+        T Multiply(T a, T b);
         /// <summary>
-        /// dst = a / b
+        /// a / b
         /// </summary>
         /// <param name="a"></param>
         /// <param name="b"></param>
-        /// <param name="dst"></param>
-        static abstract void Divide(in TA a, in TB b, ref TC dst);
+        T Divide(T a, T b);
         /// <summary>
-        /// dst = a % b
+        /// a % b
         /// </summary>
         /// <param name="a"></param>
         /// <param name="b"></param>
-        /// <param name="dst"></param>
-        static abstract void Module(in TA a, in TB b, ref TC dst);
+        T Module(T a, T b);
         /// <summary>
-        /// dst += a * b
+        /// a * b + c
         /// </summary>
         /// <param name="a"></param>
         /// <param name="b"></param>
         /// <param name="dst"></param>
-        static abstract TC MultiplyAndAdd(TA a, TB b, TC dst);
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        T MultiplyAndAdd(T a, T b, T c);
     }
 }
diff --git a/NN/Abstraction/DType/IUnaryDTypeHandler.cs b/NN/Abstraction/DType/IUnaryDTypeHandler.cs
index cfc24d5..d65ce72 100644
--- a/NN/Abstraction/DType/IUnaryDTypeHandler.cs
+++ b/NN/Abstraction/DType/IUnaryDTypeHandler.cs
@@ -8,11 +8,7 @@ namespace NN.Native.Abstraction.DType
 {
     public interface IUnaryDTypeHandler<T> where T: unmanaged
     {
-        //static abstract void LeftShiftInplace(ref T src, int offset);
-        //static abstract void RightShiftAddInplace(ref T src, int offset);
-        //static abstract void Increase(ref T src);
-        //static abstract void Decrease(ref T src);
-        static abstract T Zero { get; }
-        static abstract T One { get; }
+        T Zero { get; }
+        T One { get; }
     }
 }
diff --git a/NN/Abstraction/Operators/IMatmulOperator.cs b/NN/Abstraction/Operators/IMatmulOperator.cs
index c2e36dc..702a6a2 100644
--- a/NN/Abstraction/Operators/IMatmulOperator.cs
+++ b/NN/Abstraction/Operators/IMatmulOperator.cs
@@ -2,8 +2,12 @@
 
 namespace NN.Native.Abstraction.Operators
 {
-    public interface IMatmulOperator<TA, TB, TC>: ITernaryOperator where TA : unmanaged where TB : unmanaged where TC : unmanaged
+    public interface IMatmulOperator<T>: ITernaryOperator where T : unmanaged
     {
-        static abstract void Exec(ReadOnlySpan<TA> a, ReadOnlySpan<TB> b, Span<TC> c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
+#if NET7_0_OR_GREATER
+        static abstract void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
+#else
+        void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
+#endif
     }
 }
diff --git a/NN/Abstraction/Operators/IOperator.cs b/NN/Abstraction/Operators/IOperator.cs
index 354b1f0..c2fa746 100644
--- a/NN/Abstraction/Operators/IOperator.cs
+++ b/NN/Abstraction/Operators/IOperator.cs
@@ -8,7 +8,12 @@ namespace NN.Native.Abstraction.Operators
 {
     public interface IOperator
     {
+#if NET7_0_OR_GREATER
         static abstract bool IsThreadSafe { get; }
         static abstract OperatorHandlerType HandlerType { get; }
+#else
+        bool IsThreadSafe { get; }
+        OperatorHandlerType HandlerType { get; }
+#endif
     }
 }
diff --git a/NN/Abstraction/Operators/ITypeConvertOperator.cs b/NN/Abstraction/Operators/ITypeConvertOperator.cs
index 200f7f5..ef39c37 100644
--- a/NN/Abstraction/Operators/ITypeConvertOperator.cs
+++ b/NN/Abstraction/Operators/ITypeConvertOperator.cs
@@ -6,6 +6,10 @@ namespace NN.Native.Abstraction.Operators
 {
     public interface ITypeConvertOperator<TA, TB>: IBinaryOperator where TA: unmanaged where TB : unmanaged
     {
+#if NET7_0_OR_GREATER
         static abstract void Exec(ReadOnlySpan<TA> a, Span<TB> b, in NativeLayout layoutA, in NativeLayout layoutB, in TypeConvertParam param);
+#else
+        void Exec(ReadOnlySpan<TA> a, Span<TB> b, in NativeLayout layoutA, in NativeLayout layoutB, in TypeConvertParam param);
+#endif
     }
 }
diff --git a/NN/Basic/DType/FloatHandler.cs b/NN/Basic/DType/FloatHandler.cs
index 6fcde53..3c36548 100644
--- a/NN/Basic/DType/FloatHandler.cs
+++ b/NN/Basic/DType/FloatHandler.cs
@@ -3,41 +3,39 @@
 
 namespace NN.Native.Basic.DType
 {
-    public class FloatHandler: IUnaryDTypeHandler<float>, IBinaryDTypeHandler<float, float>, ITernaryDTypeHandler<float, float, float>, 
+    public class Float32Handler: INativeDTypeHandler<float>,
             INativeConvertible<int, float>, INativeConvertible<bool, float>, INativeConvertible<float, float>
     {
         #region unary
-        public static float Zero { get => .0f; }
-        public static float One { get => 1.0f; }
+        public float Zero { get => .0f; }
+        public float One { get => 1.0f; }
         #endregion
 
         #region binary
-        public static void AddInplace(ref float src, in float addend) => src += addend;
-        public static void SubstractInplace(ref float src, in float subtrahend) => src -= subtrahend;
-        public static void MultiplyInplace(ref float src, in float multiplicator) => src *= multiplicator;
-        public static void DivideInplace(ref float src, in float divisor) => src /= divisor;
-        public static void ModuleInplace(ref float src, in float modulus) => src %= modulus;
-        //public static void LeftShift(ref float src, ref float dst, int offset) => throw new InvalidOperationException();
-        //public static void RightShift(ref float src, ref float dst, int offset) => throw new InvalidOperationException();
-        public static bool AreEqual(in float a, in float b) => a == b;
-        public static bool IsGraeter(in float a, in float b) => a > b;
-        public static bool IsLess(in float a, in float b) => a < b;
+        public bool AreEqual(float a, float b) => a == b;
+        public bool IsGraeter(float a, float b) => a > b;
+        public bool IsLess(float a, float b) => a < b;
         #endregion
 
         #region ternary
-        public static void Add(in float a, in float b, ref float dst) => dst = a + b;
-        public static void Substract(in float a, in float b, ref float dst) => dst = a - b;
-        public static void Multiply(in float a, in float b, ref float dst) => dst = a * b;
-        public static void Divide(in float a, in float b, ref float dst) => dst = a / b;
-        public static void Module(in float a, in float b, ref float dst) => dst = a % b;
-        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static float MultiplyAndAdd(float a, float b, float dst) => dst + a * b;
+        public float Add(float a, float b) => a + b;
+        public float Substract(float a, float b) => a - b;
+        public float Multiply(float a, float b) => a * b;
+        public float Divide(float a, float b) => a / b;
+        public float Module(float a, float b) => a % b;
+        public float MultiplyAndAdd(float a, float b, float dst) => dst + a * b;
         #endregion
 
         #region convert
-        public static void Convert(in float a, ref float b) => b = a;
-        public static void Convert(in int a, ref float b) => b = a;
-        public static void Convert(in bool a, ref float b) => b = a ? .1f: .0f;
+#if NET7_0_OR_GREATER
+        public static float Convert(float a) => a;
+        public static float Convert(int a) => a;
+        public static float Convert(bool a) => a ? .1f : .0f;
+#else
+        public float Convert(float a) => a;
+        public float Convert(int a) => a;
+        public float Convert(bool a) => a ? .1f : .0f;
+#endif
         #endregion
     }
 }
diff --git a/NN/Data/DefaultNativeMemoryManager.cs b/NN/Data/DefaultNativeMemoryManager.cs
index 27f5d5e..e6fbc4e 100644
--- a/NN/Data/DefaultNativeMemoryManager.cs
+++ b/NN/Data/DefaultNativeMemoryManager.cs
@@ -14,7 +14,7 @@ public class DefaultNativeMemoryManager: INativeMemoryManager
     {
         public Memory<T> AllocateMemory<T>(int elems) where T: unmanaged
         {
-            return new(new T[elems * Marshal.SizeOf<T>()]);
+            return new(new T[elems]);
         }
     }
 }
diff --git a/NN/Data/NativeArray.cs b/NN/Data/NativeArray.cs
index 1982926..a9069bc 100644
--- a/NN/Data/NativeArray.cs
+++ b/NN/Data/NativeArray.cs
@@ -60,7 +60,7 @@ public static NativeArray<T> Normal<T>(NativeLayout layout, T minValue, T maxVal
                     var random = new System.Random();
                     for (int i = 0; i < span.Length; i++)
                     {
-                        span[i] = random.NextSingle() * ((float)(object)maxValue - (float)(object)minValue) + (float)(object)minValue;
+                        span[i] = ((float)random.NextDouble()) * ((float)(object)maxValue - (float)(object)minValue) + (float)(object)minValue;
                     }
                 }
                 else
diff --git a/NN/Data/NativeStorage.cs b/NN/Data/NativeStorage.cs
index 9c9decb..8dde00b 100644
--- a/NN/Data/NativeStorage.cs
+++ b/NN/Data/NativeStorage.cs
@@ -23,7 +23,7 @@ public NativeStorage(Memory<T> memory, int offset, bool mutable = true, bool own
         }
 
 
-        public override bool Equals([NotNullWhen(true)] object? obj)
+        public override bool Equals(object? obj)
         {
             if (obj is null or not NativeStorage<T>) return false;
             else return this == (NativeStorage<T>)obj;
diff --git a/NN/NN.Native.csproj b/NN/NN.Native.csproj
index eb9482b..fa53e07 100644
--- a/NN/NN.Native.csproj
+++ b/NN/NN.Native.csproj
@@ -1,7 +1,7 @@
 ﻿<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFrameworks>net7.0;net6.0;netcoreapp3.1;netstandard2.0;</TargetFrameworks>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <Folder Include="Operators\X86\" />
+    <PackageReference Include="System.Memory" Version="4.5.4" Condition="'$(TargetFramework)'=='netstandard2.0'"/>
   </ItemGroup>
 
 </Project>
diff --git a/NN/Operators/Naive/MatmulOperator.cs b/NN/Operators/Naive/MatmulOperator.cs
index fdbdf4b..6c4db7c 100644
--- a/NN/Operators/Naive/MatmulOperator.cs
+++ b/NN/Operators/Naive/MatmulOperator.cs
@@ -1,33 +1,57 @@
 ﻿using NN.Native.Abstraction.DType;
+using NN.Native.Abstraction.Operators;
 using NN.Native.Basic;
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading.Tasks;
 
 namespace NN.Native.Operators.Naive
 {
-    public class MatmulOperator<TA, TB, TC, THandler> where TA: unmanaged where TB: unmanaged where TC: unmanaged where THandler: ITernaryDTypeHandler<TA, TB, TC>, IUnaryDTypeHandler<TC>
+
+#if NET7_0_OR_GREATER
+    public class MatmulOperator<T>: IMatmulOperator<T> where T : unmanaged, INumberBase<T>
+    {
+        public static bool IsThreadSafe { get => false; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+    public class MatmulOperator<T, U>: IMatmulOperator<T> where T : unmanaged where U: INativeDTypeHandler<T>, new()
     {
-        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static unsafe void Exec(ReadOnlySpan<TA> a, ReadOnlySpan<TB> b, Span<TC> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        private static U _handler = new();
+        public bool IsThreadSafe { get => false; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
+#if NET7_0_OR_GREATER
+        //[MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        static
+#endif
+        public unsafe void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
         {
             // The array should be contiguous here
             int aRows = layoutA._shape[0];
             int aCols = layoutA._shape[1];
             int bCols = layoutB._shape[1];
-            for(int i = 0; i < aRows; i++)
+            for (int i = 0; i < aRows; i++)
             {
-                for(int j = 0; j < bCols; j++)
+                for (int j = 0; j < bCols; j++)
                 {
-                    var res = THandler.Zero;
-                    for(int k = 0; k < aCols; k++)
+#if NET7_0_OR_GREATER
+                    var res = T.Zero;
+#else
+                    var res = _handler.Zero;
+#endif
+                    for (int k = 0; k < aCols; k++)
                     {
-                        res = THandler.MultiplyAndAdd(a[i * layoutA._shape[1] + k], b[k * layoutB._shape[1] + j], res);
+#if NET7_0_OR_GREATER
+                        res += a[i * aCols + k] * b[k * bCols + j];
+#else
+                        res = _handler.MultiplyAndAdd(a[i * aCols + k], b[k * bCols + j], res);
+#endif
                     }
-                    c[i * layoutC._shape[1] + j] = res;
+                    c[i * bCols + j] = res;
                 }
             }
         }
diff --git a/NN/Operators/Naive/TypeConvertOperator.cs b/NN/Operators/Naive/TypeConvertOperator.cs
index 615cd39..39f5d79 100644
--- a/NN/Operators/Naive/TypeConvertOperator.cs
+++ b/NN/Operators/Naive/TypeConvertOperator.cs
@@ -7,10 +7,16 @@
 
 namespace NN.Native.Operators.Naive
 {
-    public class TypeConvertOperator<TA, TB, THandler>: TypeConvertOperatorBase where TA: unmanaged where TB: unmanaged where THandler: INativeConvertible<TA, TB>
+    public class TypeConvertOperator<TA, TB, THandler>: TypeConvertOperatorBase where TA: unmanaged where TB: unmanaged where THandler: INativeConvertible<TA, TB>, new()
     {
+#if NET7_0_OR_GREATER
         public static bool IsThreadSafe { get => false; }
         public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+        private static THandler _handler = new();
+        public bool IsThreadSafe { get => false; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
         public static unsafe void Exec(ReadOnlySpan<TA> a, Span<TB> b, in NativeLayout layoutA, in NativeLayout layoutB, in TypeConvertParam param)
         {
             int totalElems = layoutA.TotalElemCount();
@@ -20,7 +26,11 @@ public static unsafe void Exec(ReadOnlySpan<TA> a, Span<TB> b, in NativeLayout l
                 var indexEnumeratorB = NativeLayout.GetIndexEnumerator(layoutB);
                 for (int i = 0; i < totalElems; i++)
                 {
-                    THandler.Convert(a[indexEnumeratorA.MoveNext()], ref b[indexEnumeratorB.MoveNext()]);
+#if NET7_0_OR_GREATER
+                    b[indexEnumeratorB.MoveNext()] = THandler.Convert(a[indexEnumeratorA.MoveNext()]);
+#else
+                    b[indexEnumeratorB.MoveNext()] = _handler.Convert(a[indexEnumeratorA.MoveNext()]);
+#endif
                 }
             }
             else
@@ -32,7 +42,11 @@ public static unsafe void Exec(ReadOnlySpan<TA> a, Span<TB> b, in NativeLayout l
                     indices[param.DimA] ^= indices[param.DimB];
                     indices[param.DimB] ^= indices[param.DimA];
                     indices[param.DimA] ^= indices[param.DimB];
-                    THandler.Convert(a[offset], ref b[layoutB.IndicesToOffset(indices)]);
+#if NET7_0_OR_GREATER
+                    b[layoutB.IndicesToOffset(indices)] = THandler.Convert(a[offset]);
+#else
+                    b[layoutB.IndicesToOffset(indices)] = _handler.Convert(a[offset]);
+#endif
                 }
             }
         }
diff --git a/NN/Operators/X86/MatmulOperator.cs b/NN/Operators/X86/MatmulOperator.cs
new file mode 100644
index 0000000..dc38c8d
--- /dev/null
+++ b/NN/Operators/X86/MatmulOperator.cs
@@ -0,0 +1,66 @@
+﻿using NN.Native.Abstraction.Operators;
+using NN.Native.Basic;
+using NN.Native.Abstraction.DType;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+using System.Runtime.CompilerServices;
+
+namespace NN.Native.Operators.X86
+{
+#if NET7_0_OR_GREATER
+    public class MatmulOperator<T> : IMatmulOperator<T> where T : unmanaged, INumberBase<T>
+    {
+        public static bool IsThreadSafe { get => false; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+    public class MatmulOperator<T, U>: IMatmulOperator<T> where T : unmanaged where U: INativeDTypeHandler<T>, new()
+    {
+        private static U _handler = new();
+        public bool IsThreadSafe { get => false; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
+#if NET7_0_OR_GREATER
+        //[MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        static
+#endif
+        public unsafe void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        {
+            // The array should be contiguous here
+            int aRows = layoutA._shape[0];
+            int aCols = layoutA._shape[1];
+            int bCols = layoutB._shape[1];
+
+            for(int j = 0; j <= bCols - 4; j += 4)
+            {
+                for(int i = 0; i < aRows; i++)
+                {
+                    for(int k = 0; k < aCols; k++)
+                    {
+#if NET7_0_OR_GREATER
+                        c[i * bCols + j] += a[i * aCols + k] * b[k * bCols + j];
+                        c[i * bCols + j + 1] += a[i * aCols + k] * b[k * bCols + j + 1];
+                        c[i * bCols + j + 2] += a[i * aCols + k] * b[k * bCols + j + 2];
+                        c[i * bCols + j + 3] += a[i * aCols + k] * b[k * bCols + j + 3];
+#endif
+                    }
+                }
+            }
+            for(int j = bCols / 4 * 4; j < bCols; j++)
+            {
+                for (int i = 0; i < aRows; i++)
+                {
+                    for (int k = 0; k < aCols; k++)
+                    {
+#if NET7_0_OR_GREATER
+                        c[i * bCols + j] += a[i * aCols + k] * b[k * bCols + j];
+#endif
+                    }
+                }
+            }
+        }
+    }
+}

From 5a0db80df278f83f7b981cecd8f08b22ff8b40b0 Mon Sep 17 00:00:00 2001
From: AsakusaRinne <AsakusaRinne@gmail.com>
Date: Thu, 3 Nov 2022 13:32:17 +0800
Subject: [PATCH 3/5] change to unsafe pointer.

---
 Benchmark/Program.cs                        |   2 +
 NN.BenchmarkTest/MatmulBenchmark.cs         |   9 +-
 NN.BenchmarkTest/NN.BenchmarkTest.csproj    |   1 +
 NN.UnitTests/NN.UnitTests.csproj            |   2 +
 NN.UnitTests/Operators/Naive/NaiveTest.cs   |   2 +-
 NN.UnitTests/Operators/X86/MatmulTest.cs    |  14 +--
 NN/Abstraction/Operators/IMatmulOperator.cs |   6 +-
 NN/Data/NativeArray.cs                      |   6 ++
 NN/NN.Native.csproj                         |   2 +-
 NN/Operators/Naive/MatmulOperator.cs        |   4 +-
 NN/Operators/X86/MatmulOperator.cs          | 102 ++++++++++++++++++--
 11 files changed, 124 insertions(+), 26 deletions(-)

diff --git a/Benchmark/Program.cs b/Benchmark/Program.cs
index 643936f..afc379a 100644
--- a/Benchmark/Program.cs
+++ b/Benchmark/Program.cs
@@ -6,10 +6,12 @@
 using NN.Native.Basic;
 using NN.Native.Operators.Naive;
 using NN.Native.Basic.DType;
+using System.Runtime.Remoting;
 
 var summary = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
 Console.WriteLine(summary);
 
+
 public class MatmulFloat32Benchmark
 {
     public IEnumerable<int[][]> DimensionValues => new int[][][] {
diff --git a/NN.BenchmarkTest/MatmulBenchmark.cs b/NN.BenchmarkTest/MatmulBenchmark.cs
index e0bcf23..fbb25c0 100644
--- a/NN.BenchmarkTest/MatmulBenchmark.cs
+++ b/NN.BenchmarkTest/MatmulBenchmark.cs
@@ -14,6 +14,7 @@ namespace NN.BenchmarkTest
     //[SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.NetCoreApp31, baseline: true)]
     //[SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.Net60)]
     [SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.Net70)]
+    //[BenchmarkDotNet.Attributes.RyuJitX64Job]
     public class MatmulFloat32Benchmark
     {
         public IEnumerable<DimensionHelper> DimensionValues => new DimensionHelper[] {
@@ -46,13 +47,15 @@ public void NaiveMatmul()
 #endif
         }
         [Benchmark]
-        public void X86Matmul()
+        public unsafe void X86Matmul()
         {
             NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
 #if NET7_0_OR_GREATER
-            NN.Native.Operators.X86.MatmulOperator<float>.Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+            NN.Native.Operators.X86.MatmulOperator<float>.Exec((float*)ArrayLeft.Pin().Pointer, (float*)ArrayRight.Pin().Pointer
+                , (float*)res.Pin().Pointer, ArrayLeft._layout, ArrayRight._layout, res._layout);
 #else
-            new NN.Native.Operators.X86.MatmulOperator<float, Float32Handler>().Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+            new NN.Native.Operators.X86.MatmulOperator<float, Float32Handler>().Exec((float*)ArrayLeft.Pin().Pointer, (float*)ArrayRight.Pin().Pointer
+                , (float*)res.Pin().Pointer, ArrayLeft._layout, ArrayRight._layout, res._layout);
 #endif
         }
     }
diff --git a/NN.BenchmarkTest/NN.BenchmarkTest.csproj b/NN.BenchmarkTest/NN.BenchmarkTest.csproj
index 8fc8137..c4dad81 100644
--- a/NN.BenchmarkTest/NN.BenchmarkTest.csproj
+++ b/NN.BenchmarkTest/NN.BenchmarkTest.csproj
@@ -6,6 +6,7 @@
     <LangVersion>preview</LangVersion>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+    <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
   </PropertyGroup>
 
   <ItemGroup>
diff --git a/NN.UnitTests/NN.UnitTests.csproj b/NN.UnitTests/NN.UnitTests.csproj
index 53a6065..439e533 100644
--- a/NN.UnitTests/NN.UnitTests.csproj
+++ b/NN.UnitTests/NN.UnitTests.csproj
@@ -6,6 +6,8 @@
     <Nullable>enable</Nullable>
 
     <IsPackable>false</IsPackable>
+
+    <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
   </PropertyGroup>
 
   <ItemGroup>
diff --git a/NN.UnitTests/Operators/Naive/NaiveTest.cs b/NN.UnitTests/Operators/Naive/NaiveTest.cs
index b40e08e..108a1e1 100644
--- a/NN.UnitTests/Operators/Naive/NaiveTest.cs
+++ b/NN.UnitTests/Operators/Naive/NaiveTest.cs
@@ -36,7 +36,7 @@ public void MatmulTest()
             var a = NativeArray.FromArray(new NativeLayout(new int[] { 3, 4 }), new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 });
             var b = NativeArray.FromArray(new NativeLayout(new int[] { 4, 2 }), new float[] { 1, 2, 1, 2, 1, 1, 2, 3 });
             var c = new NativeArray<float>(new NativeLayout(new int[] { 3, 2 }), new DefaultNativeMemoryManager());
-            Native.Operators.X86.MatmulOperator<float>.Exec(a.Span, b.Span, c.Span, a._layout, b._layout, c._layout);
+            Native.Operators.Naive.MatmulOperator<float>.Exec(a.Span, b.Span, c.Span, a._layout, b._layout, c._layout);
             _output.WriteLine(c.Print());
         }
     }
diff --git a/NN.UnitTests/Operators/X86/MatmulTest.cs b/NN.UnitTests/Operators/X86/MatmulTest.cs
index eeeb54f..13fe273 100644
--- a/NN.UnitTests/Operators/X86/MatmulTest.cs
+++ b/NN.UnitTests/Operators/X86/MatmulTest.cs
@@ -19,16 +19,16 @@ public class MatmulTest
             };
 
         [Fact]
-        public void Float32Test()
+        public unsafe void Float32Test()
         {
             foreach(var shape in Shapes)
             {
-                var a = NativeArray.Random.Normal<float>(new NativeLayout(shape.Item1), -1000, 1000);
-                var b = NativeArray.Random.Normal<float>(new NativeLayout(shape.Item2), -1000, 1000);
-                var naiveResult = new NativeArray<float>(new NativeLayout(new int[] { shape.Item1[0], shape.Item2[1] }), new DefaultNativeMemoryManager());
-                var x86Result = new NativeArray<float>(new NativeLayout(new int[] { shape.Item1[0], shape.Item2[1] }), new DefaultNativeMemoryManager());
-                Native.Operators.Naive.MatmulOperator<float>.Exec(a.Span, b.Span, naiveResult.Span, a._layout, b._layout, naiveResult._layout);
-                Native.Operators.X86.MatmulOperator<float>.Exec(a.Span, b.Span, x86Result.Span, a._layout, b._layout, x86Result._layout);
+                var a = NativeArray.Random.Normal<int>(new NativeLayout(shape.Item1), -10, 10);
+                var b = NativeArray.Random.Normal<int>(new NativeLayout(shape.Item2), -10, 10);
+                var naiveResult = new NativeArray<int>(new NativeLayout(new int[] { shape.Item1[0], shape.Item2[1] }), new DefaultNativeMemoryManager());
+                var x86Result = new NativeArray<int>(new NativeLayout(new int[] { shape.Item1[0], shape.Item2[1] }), new DefaultNativeMemoryManager());
+                Native.Operators.Naive.MatmulOperator<int>.Exec(a.Span, b.Span, naiveResult.Span, a._layout, b._layout, naiveResult._layout);
+                Native.Operators.X86.MatmulOperator<int>.Exec((int*)a.Pin().Pointer, (int*)b.Pin().Pointer, (int*)x86Result.Pin().Pointer, a._layout, b._layout, x86Result._layout);
                 var enumeratorNaive = NativeLayout.GetIndexEnumerator(naiveResult._layout);
                 var enumeratorX86 = NativeLayout.GetIndexEnumerator(x86Result._layout);
                 var naiveSpan = naiveResult.Span;
diff --git a/NN/Abstraction/Operators/IMatmulOperator.cs b/NN/Abstraction/Operators/IMatmulOperator.cs
index 702a6a2..d6eca52 100644
--- a/NN/Abstraction/Operators/IMatmulOperator.cs
+++ b/NN/Abstraction/Operators/IMatmulOperator.cs
@@ -2,12 +2,12 @@
 
 namespace NN.Native.Abstraction.Operators
 {
-    public interface IMatmulOperator<T>: ITernaryOperator where T : unmanaged
+    public unsafe interface IMatmulOperator<T>: ITernaryOperator where T : unmanaged
     {
 #if NET7_0_OR_GREATER
-        static abstract void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
+        static abstract void Exec(T* a, T* b, T* c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
 #else
-        void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
+        void Exec(T* a, T* b, T* c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
 #endif
     }
 }
diff --git a/NN/Data/NativeArray.cs b/NN/Data/NativeArray.cs
index a9069bc..ed384bf 100644
--- a/NN/Data/NativeArray.cs
+++ b/NN/Data/NativeArray.cs
@@ -1,6 +1,7 @@
 ﻿using NN.Native.Abstraction.Data;
 using NN.Native.Basic;
 using System;
+using System.Buffers;
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.CompilerServices;
@@ -29,6 +30,11 @@ public NativeArray(NativeLayout layout, Memory<T> memory)
             _storage = new NativeStorage<T>(memory, 0);
         }
 
+        public unsafe MemoryHandle Pin()
+        {
+            return _storage._memory.Pin();
+        }
+
         public readonly bool IsMutable() => _storage._mutable;
         public readonly bool IsMemoryOwner() => _storage._owned;
     }
diff --git a/NN/NN.Native.csproj b/NN/NN.Native.csproj
index fa53e07..47a541a 100644
--- a/NN/NN.Native.csproj
+++ b/NN/NN.Native.csproj
@@ -1,7 +1,7 @@
 ﻿<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>net7.0;net6.0;netcoreapp3.1;netstandard2.0;</TargetFrameworks>
+    <TargetFrameworks>net7.0;net6.0;netcoreapp3.1;</TargetFrameworks>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
diff --git a/NN/Operators/Naive/MatmulOperator.cs b/NN/Operators/Naive/MatmulOperator.cs
index 6c4db7c..7861d2c 100644
--- a/NN/Operators/Naive/MatmulOperator.cs
+++ b/NN/Operators/Naive/MatmulOperator.cs
@@ -13,12 +13,12 @@ namespace NN.Native.Operators.Naive
 {
 
 #if NET7_0_OR_GREATER
-    public class MatmulOperator<T>: IMatmulOperator<T> where T : unmanaged, INumberBase<T>
+    public class MatmulOperator<T> where T : unmanaged, INumberBase<T>
     {
         public static bool IsThreadSafe { get => false; }
         public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
 #else
-    public class MatmulOperator<T, U>: IMatmulOperator<T> where T : unmanaged where U: INativeDTypeHandler<T>, new()
+    public class MatmulOperator<T, U> where T : unmanaged where U: INativeDTypeHandler<T>, new()
     {
         private static U _handler = new();
         public bool IsThreadSafe { get => false; }
diff --git a/NN/Operators/X86/MatmulOperator.cs b/NN/Operators/X86/MatmulOperator.cs
index dc38c8d..1e7db84 100644
--- a/NN/Operators/X86/MatmulOperator.cs
+++ b/NN/Operators/X86/MatmulOperator.cs
@@ -8,6 +8,9 @@
 using System.Text;
 using System.Threading.Tasks;
 using System.Runtime.CompilerServices;
+using System.ComponentModel.Design.Serialization;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
 
 namespace NN.Native.Operators.X86
 {
@@ -27,31 +30,90 @@ public class MatmulOperator<T> : IMatmulOperator<T> where T : unmanaged, INumber
         //[MethodImpl(MethodImplOptions.AggressiveOptimization)]
         static
 #endif
-        public unsafe void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        public unsafe void Exec(T* a, T* b, T* c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
         {
             // The array should be contiguous here
             int aRows = layoutA._shape[0];
             int aCols = layoutA._shape[1];
             int bCols = layoutB._shape[1];
 
-            for(int j = 0; j <= bCols - 4; j += 4)
+            T c00, c01, c02, c03, c10, c11, c12, c13, c20, c21, c22, c23, c30, c31, c32, c33;
+            T b00, b01, b02, b03;
+
+            for (int i = 0; i <= aRows - 4; i += 4)
+            {
+                for (int j = 0; j <= bCols - 4; j += 4)
+                {
+#if NET7_0_OR_GREATER
+                    c00 = c01 = c02 = c03 = c10 = c11 = c12 = c13 = c20 = c21 = c22 = c23 = c30 = c31 = c32 = c33 = T.Zero;
+                    b00 = b01 = b02 = b03 = T.Zero;
+#else
+                    c00 = c01 = c02 = c03 = c10 = c11 = c12 = c13 = c20 = c21 = c22 = c23 = c30 = c31 = c32 = c33 = _handler.Zero;
+                    b00 = b01 = b02 = b03 = _handler.Zero;
+#endif
+                    int idx0 = i * aCols, idx1 = (i + 1) * aCols, idx2 = (i + 2) * aCols, idx3 = (i + 3) * aCols;
+
+                    for (int k = 0; k < aCols; k++)
+                    {
+                        b00 = b[k * bCols + j];
+                        b01 = b[k * bCols + j + 1];
+                        b02 = b[k * bCols + j + 2];
+                        b03 = b[k * bCols + j + 3];
+#if NET7_0_OR_GREATER
+                        c00 += b00 * a[idx0];
+                        c10 += b00 * a[idx1];
+                        c20 += b00 * a[idx2];
+                        c30 += b00 * a[idx3];
+
+                        c01 += b01 * a[idx0];
+                        c11 += b01 * a[idx1];
+                        c21 += b01 * a[idx2];
+                        c31 += b01 * a[idx3];
+
+                        c02 += b02 * a[idx0];
+                        c12 += b02 * a[idx1];
+                        c22 += b02 * a[idx2];
+                        c32 += b02 * a[idx3];
+
+                        c03 += b03 * a[idx0++];
+                        c13 += b03 * a[idx1++];
+                        c23 += b03 * a[idx2++];
+                        c33 += b03 * a[idx3++];
+#endif
+                    }
+                    c[i * bCols + j] = c00;
+                    c[i * bCols + j + 1] = c01;
+                    c[i * bCols + j + 2] = c02;
+                    c[i * bCols + j + 3] = c03;
+                    c[(i + 1) * bCols + j] = c10;
+                    c[(i + 1) * bCols + j + 1] = c11;
+                    c[(i + 1) * bCols + j + 2] = c12;
+                    c[(i + 1) * bCols + j + 3] = c13;
+                    c[(i + 2) * bCols + j] = c20;
+                    c[(i + 2) * bCols + j + 1] = c21;
+                    c[(i + 2) * bCols + j + 2] = c22;
+                    c[(i + 2) * bCols + j + 3] = c23;
+                    c[(i + 3) * bCols + j] = c30;
+                    c[(i + 3) * bCols + j + 1] = c31;
+                    c[(i + 3) * bCols + j + 2] = c32;
+                    c[(i + 3) * bCols + j + 3] = c33;
+                }
+            }
+            for (int i = aRows / 4 * 4; i < aRows; i++)
             {
-                for(int i = 0; i < aRows; i++)
+                for (int j = 0; j < bCols; j++)
                 {
-                    for(int k = 0; k < aCols; k++)
+                    for (int k = 0; k < aCols; k++)
                     {
 #if NET7_0_OR_GREATER
                         c[i * bCols + j] += a[i * aCols + k] * b[k * bCols + j];
-                        c[i * bCols + j + 1] += a[i * aCols + k] * b[k * bCols + j + 1];
-                        c[i * bCols + j + 2] += a[i * aCols + k] * b[k * bCols + j + 2];
-                        c[i * bCols + j + 3] += a[i * aCols + k] * b[k * bCols + j + 3];
 #endif
                     }
                 }
             }
-            for(int j = bCols / 4 * 4; j < bCols; j++)
+            for (int i = 0; i < aRows / 4 * 4; i++)
             {
-                for (int i = 0; i < aRows; i++)
+                for (int j = bCols / 4 * 4; j < bCols; j++)
                 {
                     for (int k = 0; k < aCols; k++)
                     {
@@ -62,5 +124,27 @@ public unsafe void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in Nati
                 }
             }
         }
+
+#if NET7_0_OR_GREATER
+        //[MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        static
+#endif
+        public unsafe void Kernel32b8x8(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        {
+            int aRows = layoutA._shape[0];
+            int aCols = layoutA._shape[1];
+            int bCols = layoutB._shape[1];
+
+            Vector256<T> c_0_v = new Vector256<T>();
+            Vector256<T> c_1_v = new Vector256<T>();
+            Vector256<T> c_2_v = new Vector256<T>();
+            Vector256<T> c_3_v = new Vector256<T>();
+            Vector256<T> c_4_v = new Vector256<T>();
+            Vector256<T> c_5_v = new Vector256<T>();
+            Vector256<T> c_6_v = new Vector256<T>();
+            Vector256<T> c_7_v = new Vector256<T>();
+
+            
+        }
     }
 }

From d6b260c8660003878bd841fa601f3b0310cd76da Mon Sep 17 00:00:00 2001
From: AsakusaRinne <AsakusaRinne@gmail.com>
Date: Fri, 4 Nov 2022 15:57:13 +0800
Subject: [PATCH 4/5] change back to span.

---
 NN.BenchmarkTest/MatmulBenchmark.cs         |   6 +-
 NN.UnitTests/Operators/X86/MatmulTest.cs    |   2 +-
 NN/Abstraction/Operators/IMatmulOperator.cs |   4 +-
 NN/Operators/Naive/MatmulOperator.cs        |   2 +-
 NN/Operators/X86/MatmulOperator.cs          | 132 ++++++++------------
 5 files changed, 60 insertions(+), 86 deletions(-)

diff --git a/NN.BenchmarkTest/MatmulBenchmark.cs b/NN.BenchmarkTest/MatmulBenchmark.cs
index fbb25c0..c784030 100644
--- a/NN.BenchmarkTest/MatmulBenchmark.cs
+++ b/NN.BenchmarkTest/MatmulBenchmark.cs
@@ -51,11 +51,9 @@ public unsafe void X86Matmul()
         {
             NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
 #if NET7_0_OR_GREATER
-            NN.Native.Operators.X86.MatmulOperator<float>.Exec((float*)ArrayLeft.Pin().Pointer, (float*)ArrayRight.Pin().Pointer
-                , (float*)res.Pin().Pointer, ArrayLeft._layout, ArrayRight._layout, res._layout);
+            NN.Native.Operators.X86.MatmulOperator<float>.Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
 #else
-            new NN.Native.Operators.X86.MatmulOperator<float, Float32Handler>().Exec((float*)ArrayLeft.Pin().Pointer, (float*)ArrayRight.Pin().Pointer
-                , (float*)res.Pin().Pointer, ArrayLeft._layout, ArrayRight._layout, res._layout);
+            new NN.Native.Operators.X86.MatmulOperator<float, Float32Handler>().Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
 #endif
         }
     }
diff --git a/NN.UnitTests/Operators/X86/MatmulTest.cs b/NN.UnitTests/Operators/X86/MatmulTest.cs
index 13fe273..c15453c 100644
--- a/NN.UnitTests/Operators/X86/MatmulTest.cs
+++ b/NN.UnitTests/Operators/X86/MatmulTest.cs
@@ -28,7 +28,7 @@ public unsafe void Float32Test()
                 var naiveResult = new NativeArray<int>(new NativeLayout(new int[] { shape.Item1[0], shape.Item2[1] }), new DefaultNativeMemoryManager());
                 var x86Result = new NativeArray<int>(new NativeLayout(new int[] { shape.Item1[0], shape.Item2[1] }), new DefaultNativeMemoryManager());
                 Native.Operators.Naive.MatmulOperator<int>.Exec(a.Span, b.Span, naiveResult.Span, a._layout, b._layout, naiveResult._layout);
-                Native.Operators.X86.MatmulOperator<int>.Exec((int*)a.Pin().Pointer, (int*)b.Pin().Pointer, (int*)x86Result.Pin().Pointer, a._layout, b._layout, x86Result._layout);
+                Native.Operators.X86.MatmulOperator<int>.Exec(a.Span, b.Span, x86Result.Span, a._layout, b._layout, x86Result._layout);
                 var enumeratorNaive = NativeLayout.GetIndexEnumerator(naiveResult._layout);
                 var enumeratorX86 = NativeLayout.GetIndexEnumerator(x86Result._layout);
                 var naiveSpan = naiveResult.Span;
diff --git a/NN/Abstraction/Operators/IMatmulOperator.cs b/NN/Abstraction/Operators/IMatmulOperator.cs
index d6eca52..4e5abe7 100644
--- a/NN/Abstraction/Operators/IMatmulOperator.cs
+++ b/NN/Abstraction/Operators/IMatmulOperator.cs
@@ -5,9 +5,9 @@ namespace NN.Native.Abstraction.Operators
     public unsafe interface IMatmulOperator<T>: ITernaryOperator where T : unmanaged
     {
 #if NET7_0_OR_GREATER
-        static abstract void Exec(T* a, T* b, T* c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
+        static abstract void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
 #else
-        void Exec(T* a, T* b, T* c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
+        void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
 #endif
     }
 }
diff --git a/NN/Operators/Naive/MatmulOperator.cs b/NN/Operators/Naive/MatmulOperator.cs
index 7861d2c..1751b4d 100644
--- a/NN/Operators/Naive/MatmulOperator.cs
+++ b/NN/Operators/Naive/MatmulOperator.cs
@@ -13,7 +13,7 @@ namespace NN.Native.Operators.Naive
 {
 
 #if NET7_0_OR_GREATER
-    public class MatmulOperator<T> where T : unmanaged, INumberBase<T>
+    public class MatmulOperator<T>: IMatmulOperator<T> where T : unmanaged, INumberBase<T>
     {
         public static bool IsThreadSafe { get => false; }
         public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
diff --git a/NN/Operators/X86/MatmulOperator.cs b/NN/Operators/X86/MatmulOperator.cs
index 1e7db84..8a96502 100644
--- a/NN/Operators/X86/MatmulOperator.cs
+++ b/NN/Operators/X86/MatmulOperator.cs
@@ -30,76 +30,26 @@ public class MatmulOperator<T> : IMatmulOperator<T> where T : unmanaged, INumber
         //[MethodImpl(MethodImplOptions.AggressiveOptimization)]
         static
 #endif
-        public unsafe void Exec(T* a, T* b, T* c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        public unsafe void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
         {
             // The array should be contiguous here
             int aRows = layoutA._shape[0];
             int aCols = layoutA._shape[1];
             int bCols = layoutB._shape[1];
 
-            T c00, c01, c02, c03, c10, c11, c12, c13, c20, c21, c22, c23, c30, c31, c32, c33;
-            T b00, b01, b02, b03;
+            int aIdx = 0, bIdx = 0;
 
-            for (int i = 0; i <= aRows - 4; i += 4)
+            for (int i = 0; i <= aRows - 8; i += 8)
             {
-                for (int j = 0; j <= bCols - 4; j += 4)
+                bIdx = 0;
+                for (int j = 0; j <= bCols - 8; j += 8)
                 {
-#if NET7_0_OR_GREATER
-                    c00 = c01 = c02 = c03 = c10 = c11 = c12 = c13 = c20 = c21 = c22 = c23 = c30 = c31 = c32 = c33 = T.Zero;
-                    b00 = b01 = b02 = b03 = T.Zero;
-#else
-                    c00 = c01 = c02 = c03 = c10 = c11 = c12 = c13 = c20 = c21 = c22 = c23 = c30 = c31 = c32 = c33 = _handler.Zero;
-                    b00 = b01 = b02 = b03 = _handler.Zero;
-#endif
-                    int idx0 = i * aCols, idx1 = (i + 1) * aCols, idx2 = (i + 2) * aCols, idx3 = (i + 3) * aCols;
-
-                    for (int k = 0; k < aCols; k++)
-                    {
-                        b00 = b[k * bCols + j];
-                        b01 = b[k * bCols + j + 1];
-                        b02 = b[k * bCols + j + 2];
-                        b03 = b[k * bCols + j + 3];
-#if NET7_0_OR_GREATER
-                        c00 += b00 * a[idx0];
-                        c10 += b00 * a[idx1];
-                        c20 += b00 * a[idx2];
-                        c30 += b00 * a[idx3];
-
-                        c01 += b01 * a[idx0];
-                        c11 += b01 * a[idx1];
-                        c21 += b01 * a[idx2];
-                        c31 += b01 * a[idx3];
-
-                        c02 += b02 * a[idx0];
-                        c12 += b02 * a[idx1];
-                        c22 += b02 * a[idx2];
-                        c32 += b02 * a[idx3];
-
-                        c03 += b03 * a[idx0++];
-                        c13 += b03 * a[idx1++];
-                        c23 += b03 * a[idx2++];
-                        c33 += b03 * a[idx3++];
-#endif
-                    }
-                    c[i * bCols + j] = c00;
-                    c[i * bCols + j + 1] = c01;
-                    c[i * bCols + j + 2] = c02;
-                    c[i * bCols + j + 3] = c03;
-                    c[(i + 1) * bCols + j] = c10;
-                    c[(i + 1) * bCols + j + 1] = c11;
-                    c[(i + 1) * bCols + j + 2] = c12;
-                    c[(i + 1) * bCols + j + 3] = c13;
-                    c[(i + 2) * bCols + j] = c20;
-                    c[(i + 2) * bCols + j + 1] = c21;
-                    c[(i + 2) * bCols + j + 2] = c22;
-                    c[(i + 2) * bCols + j + 3] = c23;
-                    c[(i + 3) * bCols + j] = c30;
-                    c[(i + 3) * bCols + j + 1] = c31;
-                    c[(i + 3) * bCols + j + 2] = c32;
-                    c[(i + 3) * bCols + j + 3] = c33;
+                    Kernel32b8x8(a.Slice(aIdx), b.Slice(bIdx), c.Slice(i * bCols + j), aRows, aCols, bCols);
+                    bIdx += 8;
                 }
+                aIdx += 8 * aCols;
             }
-            for (int i = aRows / 4 * 4; i < aRows; i++)
+            for (int i = aRows / 8 * 8; i < aRows; i++)
             {
                 for (int j = 0; j < bCols; j++)
                 {
@@ -111,9 +61,9 @@ public unsafe void Exec(T* a, T* b, T* c, in NativeLayout layoutA, in NativeLayo
                     }
                 }
             }
-            for (int i = 0; i < aRows / 4 * 4; i++)
+            for (int i = 0; i < aRows / 8 * 8; i++)
             {
-                for (int j = bCols / 4 * 4; j < bCols; j++)
+                for (int j = bCols / 8 * 8; j < bCols; j++)
                 {
                     for (int k = 0; k < aCols; k++)
                     {
@@ -125,26 +75,52 @@ public unsafe void Exec(T* a, T* b, T* c, in NativeLayout layoutA, in NativeLayo
             }
         }
 
-#if NET7_0_OR_GREATER
-        //[MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        static
-#endif
-        public unsafe void Kernel32b8x8(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        public static unsafe void Kernel32b8x8(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, int aRows, int aCols, int bCols)
         {
-            int aRows = layoutA._shape[0];
-            int aCols = layoutA._shape[1];
-            int bCols = layoutB._shape[1];
+            Vector<T> c_0_v = new Vector<T>();
+            Vector<T> c_1_v = new Vector<T>();
+            Vector<T> c_2_v = new Vector<T>();
+            Vector<T> c_3_v = new Vector<T>();
+            Vector<T> c_4_v = new Vector<T>();
+            Vector<T> c_5_v = new Vector<T>();
+            Vector<T> c_6_v = new Vector<T>();
+            Vector<T> c_7_v = new Vector<T>();
+
+            Vector<T> a_0_v, a_1_v, a_2_v, a_3_v, a_4_v, a_5_v, a_6_v, a_7_v;
+            Vector<T> b_v;
+            int offset = 0;
 
-            Vector256<T> c_0_v = new Vector256<T>();
-            Vector256<T> c_1_v = new Vector256<T>();
-            Vector256<T> c_2_v = new Vector256<T>();
-            Vector256<T> c_3_v = new Vector256<T>();
-            Vector256<T> c_4_v = new Vector256<T>();
-            Vector256<T> c_5_v = new Vector256<T>();
-            Vector256<T> c_6_v = new Vector256<T>();
-            Vector256<T> c_7_v = new Vector256<T>();
+            for(int k = 0; k < aCols; k++)
+            {
+                a_0_v = new Vector<T>(a[k]);
+                a_1_v = new Vector<T>(a[aCols + k]);
+                a_2_v = new Vector<T>(a[2 * aCols + k]);
+                a_3_v = new Vector<T>(a[3 * aCols + k]);
+                a_4_v = new Vector<T>(a[4 * aCols + k]);
+                a_5_v = new Vector<T>(a[5 * aCols + k]);
+                a_6_v = new Vector<T>(a[6 * aCols + k]);
+                a_7_v = new Vector<T>(a[7 * aCols + k]);
 
-            
+                b_v = new Vector<T>(b.Slice(offset));
+                offset += bCols;
+
+                c_0_v += a_0_v * b_v;
+                c_1_v += a_1_v * b_v;
+                c_2_v += a_2_v * b_v;
+                c_3_v += a_3_v * b_v;
+                c_4_v += a_4_v * b_v;
+                c_5_v += a_5_v * b_v;
+                c_6_v += a_6_v * b_v;
+                c_7_v += a_7_v * b_v;
+            }
+            c_0_v.CopyTo(c);
+            c_1_v.CopyTo(c.Slice(bCols));
+            c_2_v.CopyTo(c.Slice(2 * bCols));
+            c_3_v.CopyTo(c.Slice(3 * bCols));
+            c_4_v.CopyTo(c.Slice(4 * bCols));
+            c_5_v.CopyTo(c.Slice(5 * bCols));
+            c_6_v.CopyTo(c.Slice(6 * bCols));
+            c_7_v.CopyTo(c.Slice(7 * bCols));
         }
     }
 }

From f235a211ca7713d0b9c9eaeb9a0f328968365554 Mon Sep 17 00:00:00 2001
From: AsakusaRinne <AsakusaRinne@gmail.com>
Date: Thu, 24 Nov 2022 03:47:19 +0800
Subject: [PATCH 5/5] Add wrapping of NDArray.

---
 Benchmark/GenericMathRegression.cs            | 115 +++
 ...nchmark.csproj => LinearRegression.csproj} |   5 +-
 Benchmark/NumSharpRegression.cs               | 125 ++++
 Benchmark/Program.cs                          |  46 +-
 Benchmark/TFNetRegression.cs                  | 119 +++
 NN.BenchmarkTest/AddBenchmark.cs              |  62 ++
 NN.BenchmarkTest/MatmulBenchmark.cs           |  40 +-
 NN.BenchmarkTest/NN.BenchmarkTest.csproj.user |   6 +
 .../comparison/MatmulBenchmark.cs             |  65 +-
 NN.Core/Algebra/Matmul.cs                     |  34 +
 NN.Core/Arithmetic/BasicArithmetic.cs         | 144 ++++
 NN.Core/Class1.cs                             |   7 -
 NN.Core/Exceptions/MismatchedTypeException.cs |  14 +
 NN.Core/Manipulation/Transpose.cs             |  30 +
 NN.Core/Manipulation/TypeConvert.cs           |  31 +
 NN.Core/NDArray.Algebra.cs                    |  29 +
 NN.Core/NDArray.Arithmetic.cs                 | 154 ++++
 NN.Core/NDArray.Manipulation.cs               |  31 +
 NN.Core/NDArray.Math.cs                       |  41 ++
 NN.Core/NDArray.Random.cs                     |  21 +
 NN.Core/NDArray.Statistics.cs                 |  93 +++
 NN.Core/NDArray.cs                            |  77 ++
 NN.Core/NDArrayWithHandler.cs                 |  31 +
 NN.Core/NN.Core.csproj                        |   3 +-
 NN.Core/NN.Core.csproj.user                   |   6 +
 NN.Core/Numpy/NDArray.cs                      |  39 +
 NN.Core/Statistics/ElemWise.cs                |  61 ++
 NN.Core/Statistics/Reduce.cs                  |  61 ++
 NN.UnitTests/NN.UnitTests.csproj              |   3 +-
 NN.UnitTests/Operators/Naive/NaiveTest.cs     | 142 +++-
 NN.UnitTests/Operators/X86/ArithmeticTest.cs  | 266 +++++++
 NN.UnitTests/Operators/X86/MatmulTest.cs      |  49 +-
 NN.sln                                        |   2 +-
 NN/Abstraction/Common/Params.cs               |  34 +
 NN/Abstraction/DType/IBinaryDTypeHandler.cs   |   2 +
 NN/Abstraction/DType/ITernaryDTypeHandler.cs  |   2 +-
 NN/Abstraction/Data/INativeMemoryManager.cs   |   3 +
 .../Operators/Arithmetic/IAddOperator.cs      |  32 +
 .../Operators/Arithmetic/IDivideOperator.cs   |  36 +
 .../Operators/Arithmetic/IMultiplyOperator.cs |  32 +
 .../Operators/Arithmetic/INegativeOperator.cs |  25 +
 .../Operators/Arithmetic/ISubtractOperator.cs |  36 +
 .../Operators/IBinaryElemWiseOperator.cs      |  25 +
 NN/Abstraction/Operators/IDotOperator.cs      |  31 +
 NN/Abstraction/Operators/IMatmulOperator.cs   |  64 +-
 NN/Abstraction/Operators/IOperator.cs         |   2 +
 NN/Abstraction/Operators/IReduceOperator.cs   | 166 +++++
 .../Operators/ITernaryElemWiseOperator.cs     |  41 ++
 .../Operators/ITransposeOperator.cs           |  30 +
 .../Operators/ITypeConvertOperator.cs         |  23 +-
 .../Operators/IUnaryElemWiseOperator.cs       |  18 +
 .../{FloatHandler.cs => Float32Handler.cs}    |  19 +-
 NN/Basic/DType/Float64Handler.cs              |  67 ++
 NN/Basic/DType/Int32Handler.cs                |  88 ++-
 NN/Basic/NativeLayout.cs                      | 180 ++++-
 NN/Data/NativeArray.cs                        | 131 +++-
 NN/Data/NativeStorage.cs                      |   2 +-
 NN/Exceptions/BadTypeException.cs             |  14 +
 NN/Exceptions/InvalidShapeException.cs        |  14 +
 NN/Extensions/ArrayEqualExtension.cs          |  50 ++
 NN/Extensions/IndexOffsetConvertExtension.cs  |  26 +
 NN/Extensions/SpanExtension.cs                |  35 +
 NN/NN.Native.csproj                           |  14 +-
 NN/Operators/Common/Params/MatmulParam.cs     |  14 -
 .../Common/Params/TypeConvertParam.cs         |  15 -
 .../Common/TypeConvertOperatorBase.cs         |  19 -
 NN/Operators/Naive/BinaryElemWiseOperator.cs  |  50 ++
 NN/Operators/Naive/MatmulOperator.cs          |  21 +-
 NN/Operators/Naive/ReduceOperator.cs          | 106 +++
 NN/Operators/Naive/TernaryElemWiseOperator.cs |  94 +++
 NN/Operators/Naive/TransposeOperator.cs       |  57 ++
 NN/Operators/Naive/TypeConvertOperator.cs     |  22 +-
 NN/Operators/Naive/UnaryElemWiseOperator.cs   |  49 ++
 NN/Operators/X86/Arithmetic/AddOperator.cs    | 114 +++
 NN/Operators/X86/Arithmetic/DivideOperator.cs | 152 ++++
 .../X86/Arithmetic/MultiplyOperator.cs        | 113 +++
 .../X86/Arithmetic/NegativeOperator.cs        |  76 ++
 .../X86/Arithmetic/SubtractOperator.cs        | 152 ++++
 NN/Operators/X86/DotOperator.cs               |  15 +
 NN/Operators/X86/MatmulOperator.cs            | 691 ++++++++++++++++--
 New Text Document.txt                         |   6 +
 iris.data                                     | 150 ++++
 82 files changed, 4817 insertions(+), 293 deletions(-)
 create mode 100644 Benchmark/GenericMathRegression.cs
 rename Benchmark/{Benchmark.csproj => LinearRegression.csproj} (63%)
 create mode 100644 Benchmark/NumSharpRegression.cs
 create mode 100644 Benchmark/TFNetRegression.cs
 create mode 100644 NN.BenchmarkTest/AddBenchmark.cs
 create mode 100644 NN.BenchmarkTest/NN.BenchmarkTest.csproj.user
 create mode 100644 NN.Core/Algebra/Matmul.cs
 create mode 100644 NN.Core/Arithmetic/BasicArithmetic.cs
 delete mode 100644 NN.Core/Class1.cs
 create mode 100644 NN.Core/Exceptions/MismatchedTypeException.cs
 create mode 100644 NN.Core/Manipulation/Transpose.cs
 create mode 100644 NN.Core/Manipulation/TypeConvert.cs
 create mode 100644 NN.Core/NDArray.Algebra.cs
 create mode 100644 NN.Core/NDArray.Arithmetic.cs
 create mode 100644 NN.Core/NDArray.Manipulation.cs
 create mode 100644 NN.Core/NDArray.Math.cs
 create mode 100644 NN.Core/NDArray.Random.cs
 create mode 100644 NN.Core/NDArray.Statistics.cs
 create mode 100644 NN.Core/NDArray.cs
 create mode 100644 NN.Core/NDArrayWithHandler.cs
 create mode 100644 NN.Core/NN.Core.csproj.user
 create mode 100644 NN.Core/Numpy/NDArray.cs
 create mode 100644 NN.Core/Statistics/ElemWise.cs
 create mode 100644 NN.Core/Statistics/Reduce.cs
 create mode 100644 NN.UnitTests/Operators/X86/ArithmeticTest.cs
 create mode 100644 NN/Abstraction/Common/Params.cs
 create mode 100644 NN/Abstraction/Operators/Arithmetic/IAddOperator.cs
 create mode 100644 NN/Abstraction/Operators/Arithmetic/IDivideOperator.cs
 create mode 100644 NN/Abstraction/Operators/Arithmetic/IMultiplyOperator.cs
 create mode 100644 NN/Abstraction/Operators/Arithmetic/INegativeOperator.cs
 create mode 100644 NN/Abstraction/Operators/Arithmetic/ISubtractOperator.cs
 create mode 100644 NN/Abstraction/Operators/IBinaryElemWiseOperator.cs
 create mode 100644 NN/Abstraction/Operators/IDotOperator.cs
 create mode 100644 NN/Abstraction/Operators/IReduceOperator.cs
 create mode 100644 NN/Abstraction/Operators/ITernaryElemWiseOperator.cs
 create mode 100644 NN/Abstraction/Operators/ITransposeOperator.cs
 create mode 100644 NN/Abstraction/Operators/IUnaryElemWiseOperator.cs
 rename NN/Basic/DType/{FloatHandler.cs => Float32Handler.cs} (53%)
 create mode 100644 NN/Basic/DType/Float64Handler.cs
 create mode 100644 NN/Exceptions/BadTypeException.cs
 create mode 100644 NN/Exceptions/InvalidShapeException.cs
 create mode 100644 NN/Extensions/ArrayEqualExtension.cs
 create mode 100644 NN/Extensions/SpanExtension.cs
 delete mode 100644 NN/Operators/Common/Params/MatmulParam.cs
 delete mode 100644 NN/Operators/Common/Params/TypeConvertParam.cs
 delete mode 100644 NN/Operators/Common/TypeConvertOperatorBase.cs
 create mode 100644 NN/Operators/Naive/BinaryElemWiseOperator.cs
 create mode 100644 NN/Operators/Naive/ReduceOperator.cs
 create mode 100644 NN/Operators/Naive/TernaryElemWiseOperator.cs
 create mode 100644 NN/Operators/Naive/TransposeOperator.cs
 create mode 100644 NN/Operators/Naive/UnaryElemWiseOperator.cs
 create mode 100644 NN/Operators/X86/Arithmetic/AddOperator.cs
 create mode 100644 NN/Operators/X86/Arithmetic/DivideOperator.cs
 create mode 100644 NN/Operators/X86/Arithmetic/MultiplyOperator.cs
 create mode 100644 NN/Operators/X86/Arithmetic/NegativeOperator.cs
 create mode 100644 NN/Operators/X86/Arithmetic/SubtractOperator.cs
 create mode 100644 NN/Operators/X86/DotOperator.cs
 create mode 100644 New Text Document.txt
 create mode 100644 iris.data

diff --git a/Benchmark/GenericMathRegression.cs b/Benchmark/GenericMathRegression.cs
new file mode 100644
index 0000000..472432e
--- /dev/null
+++ b/Benchmark/GenericMathRegression.cs
@@ -0,0 +1,115 @@
+﻿using BenchmarkDotNet.Attributes;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Core;
+
+namespace LinearRegression
+{
+    public class NativeArrayRegressionTest
+    {
+        public NDArray<double> data;
+        public NDArray<double> label;
+        public Model model;
+        public static double splitRate = 0.8;
+        public static int recordInterval = 10;
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            (data, label) = IrisLoader.Load(@"");
+            model = new Model(BenchmarkParam.FeatureCount);
+            model.InitializeParameters();
+        }
+
+        [Benchmark]
+        public void TrainGenericMath()
+        {
+            model.Train(data, label, BenchmarkParam.TrainEpochs, 0.001, recordInterval);
+        }
+
+        public static class IrisLoader
+        {
+            public static Dictionary<string, double> mapping;
+            static IrisLoader()
+            {
+                mapping = new Dictionary<string, double>();
+                mapping.Add("Iris-setosa", 0);
+                mapping.Add("Iris-versicolor", 1);
+                mapping.Add("Iris-virginica", 2);
+            }
+            public static (NDArray<double>, NDArray<double>) Load(string path)
+            {
+                var data = NDArray<double>.Random.Normal(new int[] { BenchmarkParam.FeatureCount, BenchmarkParam.DataLength }, 0, 2);
+                var label = NDArray<double>.Random.Normal(new int[] { 1, BenchmarkParam.DataLength }, 0, 1);
+                return (data, label);
+            }
+        }
+        public static class Sigmoid
+        {
+            public static NDArray<double> Run(NDArray<double> src)
+            {
+                return 1 / (1 + NDArray<double>.Pow(Math.E, -src));
+            }
+        }
+
+        public class Model
+        {
+            private int _dataDim;
+            public NDArray<double> w;
+            public double b;
+            public Model(int dataDim)
+            {
+                _dataDim = dataDim;
+                InitializeParameters();
+            }
+            public (NDArray<double>, double) InitializeParameters()
+            {
+                w = NDArray<double>.Random.Normal(new int[] { _dataDim, 1 }, 0, 0.01);
+                b = 0;
+                return (w, b);
+            }
+            public (NDArray<double>, NDArray<double>, NDArray<double>) ForwardAndBackwardPropagate(in NDArray<double> data, in NDArray<double> label)
+            {
+                var dataNum = data.Shape[0];
+                // forward propagation
+                var z = w.Transpose(0, 1) * data + b;
+                var predict = Sigmoid.Run(z);
+                var diff = predict - label;
+
+                var cost = (-(label.Multiply(NDArray<double>.Log2(predict)) + (1 - label).Multiply(NDArray<double>.Log2(1 - predict)))).Mean();
+
+                // back propagation
+                var dw = data * diff.Transpose(0, 1) / dataNum;
+                var db = diff.Sum() / dataNum;
+                return (cost, dw, db);
+            }
+
+            public NDArray<double> UpdateParameters(NDArray<double> data, NDArray<double> label, double lr)
+            {
+                var (cost, gradW, gradB) = ForwardAndBackwardPropagate(data, label);
+
+                w -= lr * gradW;
+                b -= lr * gradB[0, 0];
+
+                return cost;
+            }
+
+            public List<double> Train(NDArray<double> data, NDArray<double> label, int epochs, double lr, int recordInterval = 5)
+            {
+                var costs = new List<double>();
+                for (int i = 1; i <= epochs; i++)
+                {
+                    var cost = UpdateParameters(data, label, lr);
+                    if (i % recordInterval == 0)
+                    {
+                        costs.Add(cost[0, 0]);
+                    }
+                }
+                return costs;
+            }
+        }
+    }
+}
diff --git a/Benchmark/Benchmark.csproj b/Benchmark/LinearRegression.csproj
similarity index 63%
rename from Benchmark/Benchmark.csproj
rename to Benchmark/LinearRegression.csproj
index 612e80d..763165b 100644
--- a/Benchmark/Benchmark.csproj
+++ b/Benchmark/LinearRegression.csproj
@@ -11,10 +11,13 @@
 
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" Version="0.13.2" />
+    <PackageReference Include="NumSharp" Version="0.30.0" />
+    <PackageReference Include="SciSharp.TensorFlow.Redist" Version="2.7.0" />
+    <PackageReference Include="TensorFlow.NET" Version="0.70.1" />
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\NN\NN.Native.csproj" />
+    <ProjectReference Include="..\NN.Core\NN.Core.csproj" />
   </ItemGroup>
 
 </Project>
diff --git a/Benchmark/NumSharpRegression.cs b/Benchmark/NumSharpRegression.cs
new file mode 100644
index 0000000..e7fb658
--- /dev/null
+++ b/Benchmark/NumSharpRegression.cs
@@ -0,0 +1,125 @@
+﻿using BenchmarkDotNet.Attributes;
+using NumSharp;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace LinearRegression
+{
+    public class NumSharpRegressionTest
+    {
+        public NDArray data;
+        public NDArray label;
+        public static double splitRate = 0.8;
+        public static int recordInterval = 10;
+        Model model;
+        [GlobalSetup]
+        public void Setup()
+        {
+            (data, label) = IrisLoader.Load(@"");
+            model = new Model(BenchmarkParam.FeatureCount);
+            model.InitializeParameters();
+        }
+        [Benchmark]
+        public void TrainNumSharp()
+        {
+            model.Train(data, label, BenchmarkParam.TrainEpochs, 0.001, recordInterval);
+        }
+        public static class IrisLoader
+        {
+            public static Dictionary<string, double> mapping;
+            static IrisLoader()
+            {
+                mapping = new Dictionary<string, double>();
+                mapping.Add("Iris-setosa", 0);
+                mapping.Add("Iris-versicolor", 1);
+                mapping.Add("Iris-virginica", 2);
+            }
+            public static (NDArray, NDArray) Load(string path)
+            {
+                NDArray data = np.random.normal(0, 2, new int[] { BenchmarkParam.FeatureCount, BenchmarkParam.DataLength });
+                NDArray label = np.random.normal(0, 1, new int[] { 1, BenchmarkParam.DataLength });
+                return (data, label);
+            }
+        }
+        public static class Sigmoid
+        {
+            public static NDArray Run(NDArray src)
+            {
+                return 1 / (1 - src);
+            }
+        }
+
+        public class Model
+        {
+            private int _dataDim;
+            public NDArray w;
+            public double b;
+            public Model(int dataDim)
+            {
+                _dataDim = dataDim;
+                InitializeParameters();
+            }
+            public (NDArray, double) InitializeParameters()
+            {
+                w = np.random.normal(0, 0.01f, new int[] { _dataDim, 1 }).astype(typeof(double));
+                b = 0;
+                return (w, b);
+            }
+            public (NDArray, NDArray, NDArray) ForwardAndBackwardPropagate(NDArray data, NDArray label)
+            {
+                var dataNum = data.shape[0];
+                // forward propagation
+                var z = w.transpose(new int[] { 0, 1 }) * data + b;
+                var predict = Sigmoid.Run(z);
+                var diff = predict - label;
+
+                var cost = np.mean((label * np.log2(predict) + (1 - label) * np.log2(1 - predict)) * (-1));
+
+                // back propagation
+                var dw = data * diff.transpose(new int[] { 0, 1 }) / dataNum;
+                var db = Sum(diff) / dataNum;
+                return (cost, dw, db);
+            }
+
+            private double Sum(NDArray src)
+            {
+                double res = 0;
+                for (int i = 0; i < src.shape[0]; i++)
+                {
+                    for (int j = 0; j < src.shape[1]; j++)
+                    {
+                        res += src.GetDouble(i, j);
+                    }
+                }
+                return res;
+            }
+
+            public NDArray UpdataParameters(NDArray data, NDArray label, double lr)
+            {
+                var (cost, gradW, gradB) = ForwardAndBackwardPropagate(data, label);
+
+                w -= lr * gradW;
+                b -= lr * gradB.GetDouble(0);
+
+                return cost;
+            }
+
+            public List<double> Train(NDArray data, NDArray label, int epochs, double lr, int recordInterval = 5)
+            {
+                var costs = new List<double>();
+                for (int i = 1; i <= epochs; i++)
+                {
+                    var cost = UpdataParameters(data, label, lr);
+                    if (i % recordInterval == 0)
+                    {
+                        costs.Add(cost.GetDouble(0));
+                    }
+                }
+                return costs;
+            }
+        }
+    }
+}
diff --git a/Benchmark/Program.cs b/Benchmark/Program.cs
index afc379a..b4004cd 100644
--- a/Benchmark/Program.cs
+++ b/Benchmark/Program.cs
@@ -1,41 +1,15 @@
-﻿using BenchmarkDotNet.Running;
-using NN.Native.Operators;
-using BenchmarkDotNet.Attributes;
-using System.Reflection;
-using NN.Native.Data;
-using NN.Native.Basic;
-using NN.Native.Operators.Naive;
-using NN.Native.Basic.DType;
-using System.Runtime.Remoting;
+﻿using BenchmarkDotNet.Attributes;
+using NumSharp;
+using BenchmarkDotNet.Running;
+using System.Linq;
+using NN.Core;
 
-var summary = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
+var summary = BenchmarkRunner.Run(typeof(Program).Assembly);
 Console.WriteLine(summary);
 
-
-public class MatmulFloat32Benchmark
+public static class BenchmarkParam
 {
-    public IEnumerable<int[][]> DimensionValues => new int[][][] {
-            new int[][]{ new int[] { 3, 4 }, new int[] { 4, 5} },
-            new int[][]{ new int[] { 32, 64 }, new int[] { 64, 48} },
-            new int[][]{ new int[] { 256, 512 }, new int[] { 512, 192} },
-            new int[][]{ new int[] { 8, 1024 }, new int[] { 1024, 1} },
-        };
-    [ParamsSource(nameof(DimensionValues))]
-    public int[][] Dimensions { get; set; }
-
-    NativeArray<float> ArrayLeft { get; set; }
-    NativeArray<float> ArrayRight { get; set; }
-    [GlobalSetup(Targets = new[] { nameof(TensornetMatmul) })]
-    public void TensornetSetup()
-    {
-        ArrayLeft = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[0]), -100, 100);
-        ArrayRight = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[1]), -100, 100);
-    }
-
-    [Benchmark]
-    public void TensornetMatmul()
-    {
-        //NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
-        //MatmulOperator<float, float, float, FloatHandler>.Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
-    }
+    public static int TrainEpochs = 100;
+    public static int DataLength = 4000;
+    public static int FeatureCount = 1000;
 }
\ No newline at end of file
diff --git a/Benchmark/TFNetRegression.cs b/Benchmark/TFNetRegression.cs
new file mode 100644
index 0000000..529b148
--- /dev/null
+++ b/Benchmark/TFNetRegression.cs
@@ -0,0 +1,119 @@
+﻿using BenchmarkDotNet.Attributes;
+using static Tensorflow.Binding;
+using Tensorflow.NumPy;
+using Microsoft.Diagnostics.Tracing.Parsers.Clr;
+
+public class TFNetRegressionTest
+{
+    public NDArray data;
+    public NDArray label;
+    public static double splitRate = 0.8;
+    public static int recordInterval = 10;
+    Model model;
+    [GlobalSetup]
+    public void Setup()
+    {
+        (data, label) = IrisLoader.Load(@"");
+        model = new Model(BenchmarkParam.FeatureCount);
+        model.InitializeParameters();
+    }
+    [Benchmark]
+    public void TrainTFNet()
+    {
+        model.Train(data, label, BenchmarkParam.TrainEpochs, 0.001, recordInterval);
+    }
+    public static class IrisLoader
+    {
+        public static Dictionary<string, double> mapping;
+        static IrisLoader()
+        {
+            mapping = new Dictionary<string, double>();
+            mapping.Add("Iris-setosa", 0);
+            mapping.Add("Iris-versicolor", 1);
+            mapping.Add("Iris-virginica", 2);
+        }
+        public static (NDArray, NDArray) Load(string path)
+        {
+            NDArray data = np.random.normal(0, 2.0f, new int[] { BenchmarkParam.FeatureCount, BenchmarkParam.DataLength }).astype(Tensorflow.TF_DataType.TF_DOUBLE);
+            NDArray label = np.random.normal(0, 1.0f, new int[] { 1, BenchmarkParam.DataLength }).astype(Tensorflow.TF_DataType.TF_DOUBLE);
+            return (data, label);
+        }
+    }
+    public static class Sigmoid
+    {
+        public static NDArray Run(NDArray src)
+        {
+            return 1 / (1 - src);
+        }
+    }
+
+    public class Model
+    {
+        private int _dataDim;
+        public NDArray w;
+        public double b;
+        public Model(int dataDim)
+        {
+            _dataDim = dataDim;
+            InitializeParameters();
+        }
+        public (NDArray, double) InitializeParameters()
+        {
+            w = np.random.normal(0, 0.01f, new int[] { _dataDim, 1 }).astype(Tensorflow.TF_DataType.TF_DOUBLE);
+            b = 0;
+            return (w, b);
+        }
+        public (NDArray, NDArray, NDArray) ForwardAndBackwardPropagate(NDArray data, NDArray label)
+        {
+            var dataNum = data.shape[0];
+            // forward propagation
+            var z = tf.matmul(tf.transpose(w), data) + b;
+            var predict = Sigmoid.Run(z.numpy());
+            var diff = predict - label;
+
+            var cost = np.mean(-(np.multiply(label, np.log(predict)) + np.multiply((1 - label), np.log(1 - predict))));
+
+            // back propagation
+            var dw = tf.matmul(data, tf.transpose(diff)) / dataNum;
+            var db = np.sum(diff) / dataNum;
+            return (cost, dw.numpy(), db);
+        }
+
+        private NDArray Transpose(NDArray src)
+        {
+            NDArray res = np.zeros(new long[] { src.shape[1], src.shape[0] });
+            for (int i = 0; i < src.shape[0]; i++)
+            {
+                for (int j = 0; j < src.shape[1]; j++)
+                {
+                    res[j, i] = src[i, j];
+                }
+            }
+            return res;
+        }
+
+        public NDArray UpdataParameters(NDArray data, NDArray label, double lr)
+        {
+            var (cost, gradW, gradB) = ForwardAndBackwardPropagate(data, label);
+
+            w -= lr * gradW;
+            b -= lr * gradB[0];
+
+            return cost;
+        }
+
+        public List<double> Train(NDArray data, NDArray label, int epochs, double lr, int recordInterval = 5)
+        {
+            var costs = new List<double>();
+            for (int i = 1; i <= epochs; i++)
+            {
+                var cost = UpdataParameters(data, label, lr);
+                if (i % recordInterval == 0)
+                {
+                    costs.Add(cost[0]);
+                }
+            }
+            return costs;
+        }
+    }
+}
\ No newline at end of file
diff --git a/NN.BenchmarkTest/AddBenchmark.cs b/NN.BenchmarkTest/AddBenchmark.cs
new file mode 100644
index 0000000..a4b4ea3
--- /dev/null
+++ b/NN.BenchmarkTest/AddBenchmark.cs
@@ -0,0 +1,62 @@
+﻿using BenchmarkDotNet.Attributes;
+using NN.Native.Basic.DType;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Operators.Naive;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Jobs;
+using NN.BenchmarkTest.Helper;
+using System.Xml.Linq;
+using System.Reflection.Metadata;
+
+namespace NN.BenchmarkTest
+{
+    [MemoryDiagnoser]
+    [BenchmarkCategory("Add")]
+    [SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.NetCoreApp31, baseline: true)]
+    [SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.Net60)]
+    [SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.Net70)]
+    //[BenchmarkDotNet.Attributes.RyuJitX64Job]
+    internal class AddFloat32Benchmark
+    {
+        public IEnumerable<DimensionHelper> DimensionValues => new DimensionHelper[] {
+            new DimensionHelper(new int[] { 125, 125 }, new int[] { 125, 125} ),
+            new DimensionHelper(new int[] { 2, 939 }, new int[] { 939, 50} ),
+            new DimensionHelper( new int[] { 32, 64 }, new int[] { 64, 96} ),
+            new DimensionHelper(new int[] { 256, 512 }, new int[] { 512, 768} ),
+            new DimensionHelper(new int[] { 500, 500 }, new int[] { 500, 500} ),
+            new DimensionHelper(new int[] { 1000, 1000 }, new int[] { 1000, 1000} ),
+        };
+        [ParamsSource(nameof(DimensionValues))]
+        public DimensionHelper Dimensions { get; set; }
+
+        NativeArray<float> ArrayLeft { get; set; }
+        NativeArray<float> ArrayRight { get; set; }
+        [GlobalSetup(Targets = new[] { nameof(X86Add), nameof(NaiveAdd) })]
+        public void GlobalSetup()
+        {
+            ArrayLeft = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[0]), -100, 100);
+            ArrayRight = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[1]), -100, 100);
+        }
+        [Benchmark]
+        public void NaiveAdd()
+        {
+            NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
+#if NET7_0_OR_GREATER
+            TernaryElemWiseOperator<float, float, float>.Exec(ArrayLeft, ArrayRight, ref res, (x, y, offset) => x + y);
+#else
+            new TernaryElemWiseOperator<float, float, float>().Exec(ArrayLeft, ArrayRight, ref res, (x, y, offset) => x + y);
+#endif
+        }
+        [Benchmark]
+        public unsafe void X86Add()
+        {
+            NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
+#if NET7_0_OR_GREATER
+            Native.Operators.X86.Arithmetic.AddOperator<float>.Exec(ArrayLeft, ArrayRight, ref res);
+#else
+            new Native.Operators.X86.Arithmetic.AddOperator<float, Float32Handler>().Exec(ArrayLeft, ArrayRight, ref res);
+#endif
+        }
+    }
+}
diff --git a/NN.BenchmarkTest/MatmulBenchmark.cs b/NN.BenchmarkTest/MatmulBenchmark.cs
index c784030..04978b0 100644
--- a/NN.BenchmarkTest/MatmulBenchmark.cs
+++ b/NN.BenchmarkTest/MatmulBenchmark.cs
@@ -11,49 +11,57 @@ namespace NN.BenchmarkTest
 {
     [MemoryDiagnoser] 
     [BenchmarkCategory("Matmul")]
-    //[SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.NetCoreApp31, baseline: true)]
-    //[SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.Net60)]
+    [SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.NetCoreApp31, baseline: true)]
+    [SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.Net60)]
     [SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.Net70)]
     //[BenchmarkDotNet.Attributes.RyuJitX64Job]
     public class MatmulFloat32Benchmark
     {
         public IEnumerable<DimensionHelper> DimensionValues => new DimensionHelper[] {
-            new DimensionHelper( new int[] { 6, 8 }, new int[] { 8, 11} ),
+            //new DimensionHelper( new int[] { 6, 7 }, new int[] { 7, 11} ),
+            //new DimensionHelper( new int[] { 30, 65 }, new int[] { 65, 91} ),
+            new DimensionHelper(new int[] { 125, 125 }, new int[] { 125, 125} ),
+            ////new DimensionHelper(new int[] { 259, 511 }, new int[] { 511, 777} ),
+            ////new DimensionHelper(new int[] { 999, 998 }, new int[] { 998, 1013} ),
+            //new DimensionHelper(new int[] { 50, 939 }, new int[] { 939, 1} ),
+            new DimensionHelper(new int[] { 2, 939 }, new int[] { 939, 50} ),
+            //new DimensionHelper(new int[] { 1023, 1 }, new int[] { 1, 50} ),
+            //new DimensionHelper(new int[] { 67, 1 }, new int[] { 1, 789} ),
+
             new DimensionHelper( new int[] { 32, 64 }, new int[] { 64, 96} ),
             new DimensionHelper(new int[] { 256, 512 }, new int[] { 512, 768} ),
-            new DimensionHelper(new int[] { 1000, 1000 }, new int[] { 1000, 1000} ),
-            new DimensionHelper(new int[] { 2, 1024 }, new int[] { 1024, 1} ),
-            new DimensionHelper(new int[] { 1024, 1 }, new int[] { 2, 1024} )
+            new DimensionHelper(new int[] { 500, 500 }, new int[] { 500, 500} ),
+            //new DimensionHelper(new int[] { 1000, 1000 }, new int[] { 1000, 1000} ),
         };
         [ParamsSource(nameof(DimensionValues))]
         public DimensionHelper Dimensions { get; set; }
 
-        NativeArray<float> ArrayLeft { get; set; }
-        NativeArray<float> ArrayRight { get; set; }
+        NativeArray<double> ArrayLeft { get; set; }
+        NativeArray<double> ArrayRight { get; set; }
         [GlobalSetup(Targets = new[] { nameof(X86Matmul), nameof(NaiveMatmul) })]
         public void GlobalSetup()
         {
-            ArrayLeft = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[0]), -100, 100);
-            ArrayRight = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[1]), -100, 100);
+            ArrayLeft = NativeArray.Random.Normal<double>(new NativeLayout(Dimensions[0]), -100, 100);
+            ArrayRight = NativeArray.Random.Normal<double>(new NativeLayout(Dimensions[1]), -100, 100);
         }
         [Benchmark]
         public void NaiveMatmul()
         {
-            NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
+            NativeArray<double> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
 #if NET7_0_OR_GREATER
-            MatmulOperator<float>.Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+            MatmulOperator<double>.Exec(ArrayLeft, ArrayRight, ref res);
 #else
-            new MatmulOperator<float, Float32Handler>().Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+            new MatmulOperator<double, Float64Handler>().Exec(ArrayLeft, ArrayRight, ref res);
 #endif
         }
         [Benchmark]
         public unsafe void X86Matmul()
         {
-            NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
+            NativeArray<double> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
 #if NET7_0_OR_GREATER
-            NN.Native.Operators.X86.MatmulOperator<float>.Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+            NN.Native.Operators.X86.MatmulOperator<double>.Exec(ArrayLeft, ArrayRight, ref res);
 #else
-            new NN.Native.Operators.X86.MatmulOperator<float, Float32Handler>().Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+            new NN.Native.Operators.X86.MatmulOperator<double, Float64Handler>().Exec(ArrayLeft, ArrayRight, ref res);
 #endif
         }
     }
diff --git a/NN.BenchmarkTest/NN.BenchmarkTest.csproj.user b/NN.BenchmarkTest/NN.BenchmarkTest.csproj.user
new file mode 100644
index 0000000..7b0d513
--- /dev/null
+++ b/NN.BenchmarkTest/NN.BenchmarkTest.csproj.user
@@ -0,0 +1,6 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <ActiveDebugFramework>net7.0</ActiveDebugFramework>
+  </PropertyGroup>
+</Project>
\ No newline at end of file
diff --git a/NN.BenchmarkTest/comparison/MatmulBenchmark.cs b/NN.BenchmarkTest/comparison/MatmulBenchmark.cs
index 41829c0..1896a86 100644
--- a/NN.BenchmarkTest/comparison/MatmulBenchmark.cs
+++ b/NN.BenchmarkTest/comparison/MatmulBenchmark.cs
@@ -6,24 +6,43 @@
 using static Tensorflow.Binding;
 using NN.Native.Operators.Naive;
 using NN.Native.Basic.DType;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Jobs;
+using NN.BenchmarkTest.Helper;
+using Tensorflow.NumPy;
 
-[MemoryDiagnoser]
+// [MemoryDiagnoser]
 [BenchmarkCategory("Comparison-MatmulFloat32")]
+// [SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.Net60)]
+[SimpleJob(RunStrategy.Throughput, runtimeMoniker: RuntimeMoniker.Net70)]
 internal class MatmulFloat32Benchmark
 {
     public IEnumerable<int[][]> DimensionValues => new int[][][] {
-            new int[][]{ new int[] { 3, 4 }, new int[] { 4, 5} },
-            new int[][]{ new int[] { 32, 64 }, new int[] { 64, 48} },
-            new int[][]{ new int[] { 256, 512 }, new int[] { 512, 192} },
-            new int[][]{ new int[] { 8, 1024 }, new int[] { 1024, 1} },
+            //new int[][]{ new int[] { 8, 8 }, new int[] { 8, 8} },
+            //new int[][]{ new int[] { 20, 20 }, new int[] { 20, 20} },
+            //new int[][]{ new int[] { 64, 64 }, new int[] { 64, 64} },
+            //new int[][]{ new int[] { 128, 128 }, new int[] { 128, 128} },
+            //new int[][]{ new int[] { 192, 192 }, new int[] { 192, 192} },
+            //new int[][]{ new int[] { 256, 256 }, new int[] { 256, 256} },
+            //new int[][]{ new int[] { 384, 384 }, new int[] { 384, 384} },
+            //new int[][]{ new int[] { 512, 512 }, new int[] { 512, 512} },
+            //new int[][]{ new int[] { 800, 800 }, new int[] { 800, 800} },
+            //new int[][]{ new int[] { 1000, 1000 }, new int[] { 1000, 1000} },
+
+            new int[][]{ new int[] { 125, 125 }, new int[] { 125, 125} },
+            new int[][]{ new int[] { 2, 939 }, new int[] { 939, 50} },
+
+            new int[][]{ new int[] { 32, 64 }, new int[] { 64, 96} },
+            new int[][]{ new int[] { 256, 512 }, new int[] { 512, 768} },
+            new int[][]{ new int[] { 500, 500 }, new int[] { 500, 500} },
         };
     [ParamsSource(nameof(DimensionValues))]
     public int[][] Dimensions { get; set; }
 
     NativeArray<float> ArrayLeft { get; set; }
     NativeArray<float> ArrayRight { get; set; }
-    NumSharp.NDArray NDArrayLeft { get; set; }
-    NumSharp.NDArray NDArrayRight { get; set; }
+    //NumSharp.NDArray NDArrayLeft { get; set; }
+    //NumSharp.NDArray NDArrayRight { get; set; }
     Tensorflow.NumPy.NDArray TFNetTensorLeft { get; set; }
     Tensorflow.NumPy.NDArray TFNetTensorRight { get; set; }
     [GlobalSetup(Targets = new[] { nameof(GenericMathMatmul) })]
@@ -32,35 +51,33 @@ public void TensornetSetup()
         ArrayLeft = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[0]), -100, 100);
         ArrayRight = NativeArray.Random.Normal<float>(new NativeLayout(Dimensions[1]), -100, 100);
     }
-    [GlobalSetup(Targets = new[] { nameof(NumSharpMatmul) })]
-    public void NumSharpSetup()
-    {
-        NDArrayLeft = NumSharp.np.random.uniform(-100, 100, new NumSharp.Shape(Dimensions[0]));
-        NDArrayRight = NumSharp.np.random.uniform(-100, 100, new NumSharp.Shape(Dimensions[1]));
-    }
+    //[GlobalSetup(Targets = new[] { nameof(NumSharpMatmul) })]
+    //public void NumSharpSetup()
+    //{
+    //    NDArrayLeft = NumSharp.np.random.uniform(-100, 100, new NumSharp.Shape(Dimensions[0]));
+    //    NDArrayRight = NumSharp.np.random.uniform(-100, 100, new NumSharp.Shape(Dimensions[1]));
+    //}
     [GlobalSetup(Targets = new[] { nameof(TFNetMatmul) })]
     public void TFNetSetup()
     {
-        TFNetTensorLeft = Tensorflow.NumPy.np.random.normal(0, 100, new Tensorflow.Shape(Dimensions[0]));
-        TFNetTensorRight = Tensorflow.NumPy.np.random.normal(0, 100, new Tensorflow.Shape(Dimensions[1]));
+        TFNetTensorLeft = Tensorflow.NumPy.np.random.normal(0, 100, new Tensorflow.Shape(Dimensions[0])).astype(Tensorflow.TF_DataType.TF_FLOAT);
+        TFNetTensorRight = Tensorflow.NumPy.np.random.normal(0, 100, new Tensorflow.Shape(Dimensions[1])).astype(Tensorflow.TF_DataType.TF_FLOAT);
     }
 
     [Benchmark]
     public void GenericMathMatmul()
     {
-        //NumSharp.np.matmul(NDArrayLeft, NDArrayRight);
-        NativeArray<float> res = new(new NativeLayout(new int[] { Dimensions[0][0], Dimensions[1][1] }), new DefaultNativeMemoryManager());
 #if NET7_0_OR_GREATER
-        MatmulOperator<float>.Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+        var res = NN.Native.Operators.X86.MatmulOperator<float>.Exec(ArrayLeft, ArrayRight);
 #else
-        new MatmulOperator<float, Float32Handler>().Exec(ArrayLeft.Span, ArrayRight.Span, res.Span, ArrayLeft._layout, ArrayRight._layout, res._layout);
+        var res = new NN.Native.Operators.X86.MatmulOperator<float, Float32Handler>().Exec(ArrayLeft, ArrayRight);
 #endif
     }
-    [Benchmark]
-    public void NumSharpMatmul()
-    {
-        NumSharp.np.matmul(NDArrayLeft, NDArrayRight);
-    }
+    //[Benchmark]
+    //public void NumSharpMatmul()
+    //{
+    //    NumSharp.np.matmul(NDArrayLeft, NDArrayRight);
+    //}
     [Benchmark]
     public void TFNetMatmul()
     {
diff --git a/NN.Core/Algebra/Matmul.cs b/NN.Core/Algebra/Matmul.cs
new file mode 100644
index 0000000..28608e3
--- /dev/null
+++ b/NN.Core/Algebra/Matmul.cs
@@ -0,0 +1,34 @@
+﻿using NN.Native.Abstraction.Common;
+using NN.Native.Basic;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Numerics;
+using NN.Native.Data;
+using NN.Native.Abstraction.DType;
+using NN.Native.Abstraction.Operators;
+using NN.Native.Operators.X86;
+using NN.Native.Abstraction.Data;
+
+namespace NN.Core.Algebra
+{
+#if NET7_0_OR_GREATER
+    internal class Matmul<T> where T : unmanaged, INumber<T>
+#else
+    internal class Matmul<T, THandler> where T: unmanaged where THandler: INativeDTypeHandler<T>, new()
+#endif
+    {
+        internal static NDArray<T> Exec(NDArray<T> a, NDArray<T> b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = MatmulOperator<T>.Exec(a._nativeArray, b._nativeArray, memoryManager);
+#else
+            var res = new MatmulOperator<T, THandler>().Exec(a._nativeArray, b._nativeArray, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+    }
+}
diff --git a/NN.Core/Arithmetic/BasicArithmetic.cs b/NN.Core/Arithmetic/BasicArithmetic.cs
new file mode 100644
index 0000000..b5cde72
--- /dev/null
+++ b/NN.Core/Arithmetic/BasicArithmetic.cs
@@ -0,0 +1,144 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Data;
+using NN.Native.Abstraction.Operators;
+using NN.Native.Operators.X86;
+using NN.Native.Operators.X86.Arithmetic;
+using NN.Native.Abstraction.DType;
+
+namespace NN.Core.Arithmetic
+{
+#if NET7_0_OR_GREATER
+    internal class BasicArithmetic<T> where T : unmanaged, INumber<T>
+#else
+    internal class BasicArithmetic<T, THandler> where T: unmanaged where THandler: INativeDTypeHandler<T>, new()
+#endif
+    {
+        internal static NDArray<T> Add(NDArray<T> a, NDArray<T> b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = AddOperator<T>.Exec(a._nativeArray, b._nativeArray, memoryManager);
+#else
+            var res = new AddOperator<T, THandler>().Exec(a._nativeArray, b._nativeArray, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+        internal static NDArray<T> Add(NDArray<T> a, T b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = AddOperator<T>.Exec(a._nativeArray, b, memoryManager);
+#else
+            var res = new AddOperator<T, THandler>().Exec(a._nativeArray, b, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+        internal static NDArray<T> Add(T a, NDArray<T> b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = AddOperator<T>.Exec(b._nativeArray, a, memoryManager);
+#else
+            var res = new AddOperator<T, THandler>().Exec(b._nativeArray, a, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+
+
+        internal static NDArray<T> Subtract(NDArray<T> a, NDArray<T> b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = SubtractOperator<T>.Exec(a._nativeArray, b._nativeArray, memoryManager);
+#else
+            var res = new SubtractOperator<T, THandler>().Exec(a._nativeArray, b._nativeArray, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+        internal static NDArray<T> Subtract(NDArray<T> a, T b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = SubtractOperator<T>.Exec(a._nativeArray, b, memoryManager);
+#else
+            var res = new SubtractOperator<T, THandler>().Exec(a._nativeArray, b, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+        internal static NDArray<T> Subtract(T a, NDArray<T> b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = SubtractOperator<T>.Exec(a, b._nativeArray, memoryManager);
+#else
+            var res = new SubtractOperator<T, THandler>().Exec(a, b._nativeArray, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+
+        internal static NDArray<T> Multiply(NDArray<T> a, NDArray<T> b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = MultiplyOperator<T>.Exec(a._nativeArray, b._nativeArray, memoryManager);
+#else
+            var res = new MultiplyOperator<T, THandler>().Exec(a._nativeArray, b._nativeArray, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+        internal static NDArray<T> Multiply(NDArray<T> a, T b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = MultiplyOperator<T>.Exec(a._nativeArray, b, memoryManager);
+#else
+            var res = new MultiplyOperator<T, THandler>().Exec(a._nativeArray, b, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+        internal static NDArray<T> Multiply(T a, NDArray<T> b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = MultiplyOperator<T>.Exec(b._nativeArray, a, memoryManager);
+#else
+            var res = new MultiplyOperator<T, THandler>().Exec(b._nativeArray, a, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+
+        internal static NDArray<T> Divide(NDArray<T> a, NDArray<T> b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = DivideOperator<T>.Exec(a._nativeArray, b._nativeArray, memoryManager);
+#else
+            var res = new DivideOperator<T, THandler>().Exec(a._nativeArray, b._nativeArray, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+        internal static NDArray<T> Divide(NDArray<T> a, T b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = DivideOperator<T>.Exec(a._nativeArray, b, memoryManager);
+#else
+            var res = new DivideOperator<T, THandler>().Exec(a._nativeArray, b, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+        internal static NDArray<T> Divide(T a, NDArray<T> b, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = DivideOperator<T>.Exec(a, b._nativeArray, memoryManager);
+#else
+            var res = new DivideOperator<T, THandler>().Exec(a, b._nativeArray, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+
+        internal static NDArray<T> Negative(NDArray<T> src, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = NegativeOperator<T>.Exec(src._nativeArray, memoryManager);
+#else
+            var res = new NegativeOperator<T, THandler>().Exec(src._nativeArray, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+    }
+}
diff --git a/NN.Core/Class1.cs b/NN.Core/Class1.cs
deleted file mode 100644
index 478914c..0000000
--- a/NN.Core/Class1.cs
+++ /dev/null
@@ -1,7 +0,0 @@
-﻿namespace NN.Core
-{
-    public class Class1
-    {
-
-    }
-}
\ No newline at end of file
diff --git a/NN.Core/Exceptions/MismatchedTypeException.cs b/NN.Core/Exceptions/MismatchedTypeException.cs
new file mode 100644
index 0000000..333abfa
--- /dev/null
+++ b/NN.Core/Exceptions/MismatchedTypeException.cs
@@ -0,0 +1,14 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Core.Exceptions
+{
+    public class MismatchedTypeException: Exception
+    {
+        public MismatchedTypeException():base(){}
+        public MismatchedTypeException(string message):base(message){}
+    }
+}
diff --git a/NN.Core/Manipulation/Transpose.cs b/NN.Core/Manipulation/Transpose.cs
new file mode 100644
index 0000000..524f036
--- /dev/null
+++ b/NN.Core/Manipulation/Transpose.cs
@@ -0,0 +1,30 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Common;
+using NN.Native.Abstraction.Data;
+using NN.Native.Abstraction.Operators;
+using NN.Native.Operators.Naive;
+
+namespace NN.Core.Manipulation
+{
+#if NET7_0_OR_GREATER
+    internal class Transpose<T> where T : unmanaged, INumber<T>
+#else
+    internal class Transpose<T> where T: unmanaged
+#endif
+    {
+        internal static NDArray<T> Exec(NDArray<T> src, in TransposeParam param, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = TransposeOperator<T>.Exec(src._nativeArray, param, memoryManager);
+#else
+            var res = new TransposeOperator<T>().Exec(src._nativeArray, param, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+    }
+}
diff --git a/NN.Core/Manipulation/TypeConvert.cs b/NN.Core/Manipulation/TypeConvert.cs
new file mode 100644
index 0000000..c814bdb
--- /dev/null
+++ b/NN.Core/Manipulation/TypeConvert.cs
@@ -0,0 +1,31 @@
+﻿using NN.Native.Abstraction.DType;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Common;
+using NN.Native.Abstraction.Data;
+using NN.Native.Abstraction.Operators;
+using NN.Native.Operators.Naive;
+
+namespace NN.Core.Manipulation
+{
+#if NET7_0_OR_GREATER
+    internal class TypeConvert<TA, TB, THandler> where TA : unmanaged, INumber<TA> where TB : unmanaged, INumber<TB> where THandler : INativeConvertible<TA, TB>, new()
+#else
+    internal class TypeConvert<TA, TB, THandler> where TA : unmanaged where TB : unmanaged where THandler : INativeConvertible<TA, TB>, new()
+#endif
+    {
+        internal static NDArray<TB> Exec(NDArray<TA> src, in TypeConvertParam param, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = TypeConvertOperator<TA, TB, THandler>.Exec(src._nativeArray, param);
+#else
+            var res = new TypeConvertOperator<TA, TB, THandler>().Exec(src._nativeArray, param);
+#endif
+            return NDArray<TB>.FromNativeArray(res);
+        }
+    }
+}
diff --git a/NN.Core/NDArray.Algebra.cs b/NN.Core/NDArray.Algebra.cs
new file mode 100644
index 0000000..e584334
--- /dev/null
+++ b/NN.Core/NDArray.Algebra.cs
@@ -0,0 +1,29 @@
+﻿using NN.Core.Algebra;
+using NN.Core.Arithmetic;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Core
+{
+    public partial class NDArray<T>
+    {
+        public virtual NDArray<T> Matmul(NDArray<T> other)
+        {
+#if NET7_0_OR_GREATER
+            if (this.Ndim <= 2 && other.Ndim <= 2)
+            {
+                return Matmul<T>.Exec(this, other);
+            }
+            else
+            {
+                throw new NotImplementedException();
+            }
+#else
+            throw new NotImplementedException();
+#endif
+        }
+    }
+}
diff --git a/NN.Core/NDArray.Arithmetic.cs b/NN.Core/NDArray.Arithmetic.cs
new file mode 100644
index 0000000..ec3b0cd
--- /dev/null
+++ b/NN.Core/NDArray.Arithmetic.cs
@@ -0,0 +1,154 @@
+﻿using NN.Core.Arithmetic;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Core
+{
+    public partial class NDArray<T>
+    {
+        #region Virtual Functions
+        public virtual NDArray<T> Add(NDArray<T> other)
+        {
+#if NET7_0_OR_GREATER
+            return Arithmetic.BasicArithmetic<T>.Add(this, other);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+        public virtual NDArray<T> Add(T other)
+        {
+#if NET7_0_OR_GREATER
+            return Arithmetic.BasicArithmetic<T>.Add(this, other);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+        public virtual NDArray<T> Subtract(NDArray<T> other)
+        {
+#if NET7_0_OR_GREATER
+            return Arithmetic.BasicArithmetic<T>.Subtract(this, other);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+        public virtual NDArray<T> Subtract(T other)
+        {
+#if NET7_0_OR_GREATER
+            return Arithmetic.BasicArithmetic<T>.Subtract(this, other);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+        public virtual NDArray<T> Multiply(NDArray<T> other)
+        {
+#if NET7_0_OR_GREATER
+            return Arithmetic.BasicArithmetic<T>.Multiply(this, other);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+        public virtual NDArray<T> Multiply(T other)
+        {
+#if NET7_0_OR_GREATER
+            return Arithmetic.BasicArithmetic<T>.Multiply(this, other);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+        public virtual NDArray<T> Divide(NDArray<T> other)
+        {
+#if NET7_0_OR_GREATER
+            return Arithmetic.BasicArithmetic<T>.Divide(this, other);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+        public virtual NDArray<T> Divide(T other)
+        {
+#if NET7_0_OR_GREATER
+            return Arithmetic.BasicArithmetic<T>.Divide(this, other);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+        public virtual NDArray<T> BeDivided(T other)
+        {
+#if NET7_0_OR_GREATER
+            return Arithmetic.BasicArithmetic<T>.Divide(other, this);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+        protected virtual NDArray<T> GetNegative()
+        {
+#if NET7_0_OR_GREATER
+            return BasicArithmetic<T>.Negative(this);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+        #endregion
+
+        #region Operators
+        public static NDArray<T> operator +(NDArray<T> a, NDArray<T> b)
+        {
+            return a.Add(b);
+        }
+        public static NDArray<T> operator +(NDArray<T> a, T b)
+        {
+            return a.Add(b);
+        }
+        public static NDArray<T> operator +(T a, NDArray<T> b)
+        {
+            return b.Add(a);
+        }
+
+        public static NDArray<T> operator -(NDArray<T> src)
+        {
+            return src.GetNegative();
+        }
+
+        public static NDArray<T> operator -(NDArray<T> a, NDArray<T> b)
+        {
+            return a.Subtract(b);
+        }
+        public static NDArray<T> operator -(NDArray<T> a, T b)
+        {
+            return a.Subtract(b);
+        }
+        public static NDArray<T> operator -(T a, NDArray<T> b)
+        {
+            return -b.Subtract(a);
+        }
+
+        public static NDArray<T> operator *(NDArray<T> a, NDArray<T> b)
+        {
+            return a.Matmul(b);
+        }
+        public static NDArray<T> operator *(NDArray<T> a, T b)
+        {
+            return a.Multiply(b);
+        }
+        public static NDArray<T> operator *(T a, NDArray<T> b)
+        {
+            return b.Multiply(a);
+        }
+
+        public static NDArray<T> operator /(NDArray<T> a, NDArray<T> b)
+        {
+            return a.Divide(b);
+        }
+        public static NDArray<T> operator /(NDArray<T> a, T b)
+        {
+            return a.Divide(b);
+        }
+        public static NDArray<T> operator /(T a, NDArray<T> b)
+        {
+            return b.BeDivided(a);
+        }
+        #endregion
+    }
+}
diff --git a/NN.Core/NDArray.Manipulation.cs b/NN.Core/NDArray.Manipulation.cs
new file mode 100644
index 0000000..399f6f5
--- /dev/null
+++ b/NN.Core/NDArray.Manipulation.cs
@@ -0,0 +1,31 @@
+﻿using NN.Core.Manipulation;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Reflection;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Core
+{
+    public partial class NDArray<T>
+    {
+        public virtual NDArray<T> Transpose(int axisA, int axisB)
+        {
+#if NET7_0_OR_GREATER
+            return Transpose<T>.Exec(this, new Native.Abstraction.Common.TransposeParam(axisA, axisB));
+#else
+            throw new NotImplementedException();
+#endif
+        }
+#if NET7_0_OR_GREATER
+        public virtual NDArray<TTo> TypeConvert<TTo>(bool transpose = false, int axisA = -1, int axisB = -1) where TTo: unmanaged, INumber<TTo>
+#else
+        public virtual NDArray<TTo> TypeConvert<TTo>(bool transpose = false, int axisA = -1, int axisB = -1) where TTo: unmanaged
+#endif
+        {
+            throw new NotImplementedException();
+        }
+    }
+}
diff --git a/NN.Core/NDArray.Math.cs b/NN.Core/NDArray.Math.cs
new file mode 100644
index 0000000..703b686
--- /dev/null
+++ b/NN.Core/NDArray.Math.cs
@@ -0,0 +1,41 @@
+﻿using NN.Native.Operators.Naive;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Core
+{
+    public partial class NDArray<T>
+    {
+        public static NDArray<double> Pow(NDArray<double> a, double b)
+        {
+#if NET7_0_OR_GREATER
+            var res = TernaryElemWiseOperator<double, double, double>.Exec(a._nativeArray, b, (x, y, idx) => Math.Pow(x, y));
+#else
+            var res = new TernaryElemWiseOperator<double, double, double>().Exec(a._nativeArray, b, (x, y, idx) => Math.Pow(x, y));
+#endif
+            return NDArray<double>.FromNativeArray(res);
+        }
+        public static NDArray<double> Pow(double a, NDArray<double> b)
+        {
+#if NET7_0_OR_GREATER
+            var res = TernaryElemWiseOperator<double, double, double>.Exec(a, b._nativeArray, (x, y, idx) => Math.Pow(x, y));
+#else
+            var res = new TernaryElemWiseOperator<double, double, double>().Exec(a, b._nativeArray, (x, y, idx) => Math.Pow(x, y));
+#endif
+            return NDArray<double>.FromNativeArray(res);
+        }
+
+        public static NDArray<double> Log2(NDArray<double> a)
+        {
+#if NET7_0_OR_GREATER
+            var res = BinaryElemWiseOperator<double, double>.Exec(a._nativeArray, (x, idx) => Math.Log2(x));
+#else
+            var res = new BinaryElemWiseOperator<double, double>().Exec(a._nativeArray, (x, idx) => Math.Log2(x));
+#endif
+            return NDArray<double>.FromNativeArray(res);
+        }
+    }
+}
diff --git a/NN.Core/NDArray.Random.cs b/NN.Core/NDArray.Random.cs
new file mode 100644
index 0000000..0e3dcc1
--- /dev/null
+++ b/NN.Core/NDArray.Random.cs
@@ -0,0 +1,21 @@
+﻿using NN.Native.Data;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Core
+{
+    public partial class NDArray<T>
+    {
+        public static class Random
+        {
+            public static NDArray<T> Normal(int[] shape, T minValue, T maxValue)
+            {
+                var nativeArray = NativeArray.Random.Normal<T>(new Native.Basic.NativeLayout(shape), minValue, maxValue);
+                return new NDArray<T>(nativeArray);
+            }
+        }
+    }
+}
diff --git a/NN.Core/NDArray.Statistics.cs b/NN.Core/NDArray.Statistics.cs
new file mode 100644
index 0000000..2b5ad06
--- /dev/null
+++ b/NN.Core/NDArray.Statistics.cs
@@ -0,0 +1,93 @@
+﻿using NN.Core.Statistics;
+using NN.Native.Abstraction.Common;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Core
+{
+    public partial class NDArray<T>
+    {
+        public virtual NDArray<T> Sum(params int[] axis)
+        {
+            ReduceParam param;
+            if(axis.Length == 0)
+            {
+                int[] realAxis = new int[this.Ndim];
+                for (int i = 0; i < this.Ndim; i++) realAxis[i] = i;
+                param = new ReduceParam(realAxis);
+            }
+            else
+            {
+                param = new ReduceParam(axis);
+            }
+#if NET7_0_OR_GREATER
+            return Reduce<T>.Sum(this, param);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+
+        public virtual NDArray<T> Mean(params int[] axis)
+        {
+            ReduceParam param;
+            if (axis.Length == 0)
+            {
+                int[] realAxis = new int[this.Ndim];
+                for (int i = 0; i < this.Ndim; i++) realAxis[i] = i;
+                param = new ReduceParam(realAxis);
+            }
+            else
+            {
+                param = new ReduceParam(axis);
+            }
+#if NET7_0_OR_GREATER
+            return Reduce<T>.Mean(this, param);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+
+        public virtual NDArray<T> Max(params int[] axis)
+        {
+            ReduceParam param;
+            if (axis.Length == 0)
+            {
+                int[] realAxis = new int[this.Ndim];
+                for (int i = 0; i < this.Ndim; i++) realAxis[i] = i;
+                param = new ReduceParam(realAxis);
+            }
+            else
+            {
+                param = new ReduceParam(axis);
+            }
+#if NET7_0_OR_GREATER
+            return Reduce<T>.Max(this, param);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+
+        public virtual NDArray<T> Min(params int[] axis)
+        {
+            ReduceParam param;
+            if (axis.Length == 0)
+            {
+                int[] realAxis = new int[this.Ndim];
+                for (int i = 0; i < this.Ndim; i++) realAxis[i] = i;
+                param = new ReduceParam(realAxis);
+            }
+            else
+            {
+                param = new ReduceParam(axis);
+            }
+#if NET7_0_OR_GREATER
+            return Reduce<T>.Min(this, param);
+#else
+            throw new NotImplementedException();
+#endif
+        }
+    }
+}
diff --git a/NN.Core/NDArray.cs b/NN.Core/NDArray.cs
new file mode 100644
index 0000000..e63f1d6
--- /dev/null
+++ b/NN.Core/NDArray.cs
@@ -0,0 +1,77 @@
+﻿using NN.Core.Numpy;
+using NN.Native;
+using NN.Native.Basic;
+using NN.Native.Basic.DType;
+using NN.Native.Data;
+using System.Numerics;
+using System.Security.Authentication.ExtendedProtection;
+
+namespace NN.Core
+{
+ 
+#if NET7_0_OR_GREATER
+    public partial class NDArray<T>: NDArray where T : unmanaged, INumber<T>
+#else
+    public partial class NDArray<T>: NDArray where T: unmanaged
+#endif
+    {
+        internal NativeArray<T> _nativeArray;
+        protected bool _isValid;
+#if NET7_0_OR_GREATER
+        public NDArray(NativeArray<T> nativeArray)
+        {
+            _nativeArray = nativeArray;
+            _isValid = true;
+        }
+        public NDArray()
+        {
+            _isValid = false;
+        }
+#else
+        protected NDArray(NativeArray<T> nativeArray)
+        {
+            _nativeArray = nativeArray;
+            _isValid = true;
+        }
+#endif
+
+        public override ReadOnlySpan<int> Shape { get => _nativeArray._layout.Shape.AsSpan(0, _nativeArray._layout.Ndim); }
+        public override int Ndim { get => _nativeArray._layout.Ndim; }
+        public override Type DType { get => typeof(T); }
+        public bool IsValid { get => _isValid; }
+
+        public override object GetAt(params int[] indices)
+        {
+            return _nativeArray[indices];
+        }
+        public T this[params int[] indices]
+        {
+            get => _nativeArray[indices];
+            set => _nativeArray[indices] = value;
+        }
+
+        internal static NDArray<T> FromNativeArray(NativeArray<T> nativeArray)
+        {
+#if NET7_0_OR_GREATER
+            return new NDArray<T>(nativeArray);
+#else
+            if (typeof(T) == typeof(int))
+            {
+                return (NDArray<T>)(object)(new NDArrayWithHandler<int, Int32Handler>((NativeArray<int>)nativeArray) as NDArray<int>); 
+            }
+            else if (typeof(T) == typeof(float))
+            {
+                return (NDArray<T>)(object)(new NDArrayWithHandler<float, Float32Handler>((NativeArray<float>)nativeArray) as NDArray<float>);
+            }
+            else if (typeof(T) == typeof(double))
+            {
+                return (NDArray<T>)(object)(new NDArrayWithHandler<double, Float64Handler>((NativeArray<double>)nativeArray) as NDArray<double>);
+            }
+            else
+            {
+                throw new NotImplementedException();
+            }
+#endif
+        }
+    }
+}
\ No newline at end of file
diff --git a/NN.Core/NDArrayWithHandler.cs b/NN.Core/NDArrayWithHandler.cs
new file mode 100644
index 0000000..519af55
--- /dev/null
+++ b/NN.Core/NDArrayWithHandler.cs
@@ -0,0 +1,31 @@
+﻿using NN.Native.Abstraction.DType;
+using NN.Native.Data;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Core
+{
+    //#if !NET7_0_OR_GREATER
+    //    internal class NDArrayWithHandler<TData, THandler>: NDArray<TData> where TData : unmanaged where THandler : INativeDTypeHandler<TData>
+    //    {
+    //        public NDArrayWithHandler(NativeArray<TData> nativeArray): base(nativeArray)
+    //        {
+
+//        }
+//    }
+//#endif
+#if !NET7_0_OR_GREATER
+    internal class NDArrayWithHandler<TData, THandler> : NDArray<TData> where TData : unmanaged where THandler : INativeDTypeHandler<TData>
+    {
+        public NDArrayWithHandler(NativeArray<TData> nativeArray) : base(nativeArray)
+        {
+
+        }
+    }
+#endif
+
+}
diff --git a/NN.Core/NN.Core.csproj b/NN.Core/NN.Core.csproj
index 496b1e7..a3161c9 100644
--- a/NN.Core/NN.Core.csproj
+++ b/NN.Core/NN.Core.csproj
@@ -1,9 +1,10 @@
 ﻿<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFrameworks>net7.0; net6.0; netcoreapp3.1</TargetFrameworks>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+    <LangVersion>preview</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
diff --git a/NN.Core/NN.Core.csproj.user b/NN.Core/NN.Core.csproj.user
new file mode 100644
index 0000000..2851e40
--- /dev/null
+++ b/NN.Core/NN.Core.csproj.user
@@ -0,0 +1,6 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <ActiveDebugFramework>net6.0</ActiveDebugFramework>
+  </PropertyGroup>
+</Project>
\ No newline at end of file
diff --git a/NN.Core/Numpy/NDArray.cs b/NN.Core/Numpy/NDArray.cs
new file mode 100644
index 0000000..6a1b8af
--- /dev/null
+++ b/NN.Core/Numpy/NDArray.cs
@@ -0,0 +1,39 @@
+﻿using NN.Core.Exceptions;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Text; 
+using System.Threading.Tasks;
+
+namespace NN.Core.Numpy
+{
+    public abstract class NDArray
+    {
+        public abstract ReadOnlySpan<int> Shape { get; }
+        public abstract int Ndim { get; }
+        public abstract Type DType { get; }
+
+        public abstract object GetAt(params int[] indices);
+
+        public T At<T>(params int[] indices)
+        {
+            return (T)GetAt(indices);
+        }
+#if NET7_0_OR_GREATER
+        public NDArray<T> As<T>() where T : unmanaged, INumber<T>
+#else
+        public NDArray<T> As<T>() where T: unmanaged
+#endif
+        {
+            if(DType == typeof(T))
+            {
+                return (NDArray<T>)(object)this;
+            }
+            else
+            {
+                throw new MismatchedTypeException();
+            }
+        }
+    }
+}
diff --git a/NN.Core/Statistics/ElemWise.cs b/NN.Core/Statistics/ElemWise.cs
new file mode 100644
index 0000000..de8e716
--- /dev/null
+++ b/NN.Core/Statistics/ElemWise.cs
@@ -0,0 +1,61 @@
+﻿using NN.Native.Abstraction.Operators;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Operators.Naive;
+
+namespace NN.Core.Statistics
+{
+#if NET7_0_OR_GREATER
+    internal class UnaryElemWise<T> where T : unmanaged, INumber<T>
+#else
+    internal class UnaryElemWise<T> where T : unmanaged
+#endif
+    {
+        internal static void Exec(NDArray<T> src, Func<T, int, T> elemFunc, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            UnaryElemWiseOperator<T>.Exec(ref src._nativeArray, elemFunc);
+#else
+            new UnaryElemWiseOperator<T>().Exec(ref src._nativeArray, elemFunc);
+#endif
+        }
+    }
+
+#if NET7_0_OR_GREATER
+    internal class BinaryElemWise<TA, TB> where TA : unmanaged, INumber<TA> where TB : unmanaged, INumber<TB>
+#else
+    internal class BinaryElemWise<TA, TB> where TA : unmanaged where TB : unmanaged
+#endif
+    {
+        internal static NDArray<TB> Exec(NDArray<TA> src, Func<TA, int, TB> elemFunc, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = BinaryElemWiseOperator<TA, TB>.Exec(src._nativeArray, elemFunc);
+#else
+            var res = new BinaryElemWiseOperator<TA, TB>().Exec(src._nativeArray, elemFunc);
+#endif
+            return NDArray<TB>.FromNativeArray(res);
+        }
+    }
+
+#if NET7_0_OR_GREATER
+    internal class TernaryElemWise<TA, TB, TC> where TA : unmanaged, INumber<TA> where TB : unmanaged, INumber<TB> where TC : unmanaged, INumber<TC>
+#else
+    internal class TernaryElemWise<TA, TB, TC> where TA : unmanaged where TB : unmanaged where TC : unmanaged
+#endif
+    {
+        internal static NDArray<TC> Exec(NDArray<TA> a, NDArray<TB> b, Func<TA, TB, int, TC> elemFunc, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = TernaryElemWiseOperator<TA, TB, TC>.Exec(a._nativeArray, b._nativeArray, elemFunc);
+#else
+            var res = new TernaryElemWiseOperator<TA, TB, TC>().Exec(a._nativeArray, b._nativeArray, elemFunc);
+#endif
+            return NDArray<TC>.FromNativeArray(res);
+        }
+    }
+}
diff --git a/NN.Core/Statistics/Reduce.cs b/NN.Core/Statistics/Reduce.cs
new file mode 100644
index 0000000..5c9855e
--- /dev/null
+++ b/NN.Core/Statistics/Reduce.cs
@@ -0,0 +1,61 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Operators.Naive;
+using NN.Native.Abstraction.Operators;
+using NN.Native.Abstraction.Data;
+using NN.Native.Abstraction.Common;
+using NN.Native.Abstraction.DType;
+
+namespace NN.Core.Statistics
+{
+#if NET7_0_OR_GREATER
+    internal class Reduce<T> where T : unmanaged, INumber<T>
+#else
+    internal class Reduce<T, THandler> where T: unmanaged where THandler: INativeDTypeHandler<T>, new()
+#endif
+    {
+        internal static NDArray<T> Sum(NDArray<T> src, in ReduceParam param, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = ReduceOperator<T, ReduceSumOpHandler<T>>.Exec(src._nativeArray, param, memoryManager);
+#else
+            var res = new ReduceOperator<T, ReduceSumOpHandler<T, THandler>>().Exec(src._nativeArray, param, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+
+        internal static NDArray<T> Mean(NDArray<T> src, in ReduceParam param, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = ReduceOperator<T, ReduceMeanOpHandler<T>>.Exec(src._nativeArray, param, memoryManager);
+#else
+            var res = new ReduceOperator<T, ReduceMeanOpHandler<T, THandler>>().Exec(src._nativeArray, param, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+
+        internal static NDArray<T> Max(NDArray<T> src, in ReduceParam param, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = ReduceOperator<T, ReduceMaxOpHandler<T>>.Exec(src._nativeArray, param, memoryManager);
+#else
+            var res = new ReduceOperator<T, ReduceMaxOpHandler<T, THandler>>().Exec(src._nativeArray, param, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+
+        internal static NDArray<T> Min(NDArray<T> src, in ReduceParam param, INativeMemoryManager? memoryManager = null, OperatorHandlerType handlerType = OperatorHandlerType.Default)
+        {
+#if NET7_0_OR_GREATER
+            var res = ReduceOperator<T, ReduceMinOpHandler<T>>.Exec(src._nativeArray, param, memoryManager);
+#else
+            var res = new ReduceOperator<T, ReduceMinOpHandler<T, THandler>>().Exec(src._nativeArray, param, memoryManager);
+#endif
+            return NDArray<T>.FromNativeArray(res);
+        }
+    }
+}
diff --git a/NN.UnitTests/NN.UnitTests.csproj b/NN.UnitTests/NN.UnitTests.csproj
index 439e533..527a4c0 100644
--- a/NN.UnitTests/NN.UnitTests.csproj
+++ b/NN.UnitTests/NN.UnitTests.csproj
@@ -1,9 +1,10 @@
 ﻿<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFrameworks>net7.0;net6.0;netcoreapp3.1;</TargetFrameworks>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+    <LangVersion>preview</LangVersion>
 
     <IsPackable>false</IsPackable>
 
diff --git a/NN.UnitTests/Operators/Naive/NaiveTest.cs b/NN.UnitTests/Operators/Naive/NaiveTest.cs
index 108a1e1..202f032 100644
--- a/NN.UnitTests/Operators/Naive/NaiveTest.cs
+++ b/NN.UnitTests/Operators/Naive/NaiveTest.cs
@@ -3,8 +3,10 @@
 using NN.Native.Basic;
 using NN.Native.Operators.Naive;
 using NN.Native.Basic.DType;
-using NN.Native.Operators.Common.Params;
+using NN.Native.Abstraction.Common;
+using NN.Native.Abstraction;
 using Xunit.Abstractions;
+using NN.Native.Abstraction.Operators;
 
 namespace NN.UnitTests.Naive
 {
@@ -20,24 +22,148 @@ public NaiveTest(ITestOutputHelper output)
         public void TypeConvertTest()
         {
             var array = NativeArray.FromArray(new NativeLayout(new int[] { 3, 4 }), new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 });
-            _output.WriteLine(array.Print());
-            var convertedArray = new NativeArray<float>(new NativeLayout(new int[] { 4, 3 }), new DefaultNativeMemoryManager());
-            TypeConvertOperator<int, float, Float32Handler>.Exec(array.Span, convertedArray.Span, array._layout, convertedArray._layout, new TypeConvertParam()
+            var truth = NativeArray.FromArray(new NativeLayout(new int[] { 4, 3 }), new float[] { 1, 5, 9, 2, 6, 10, 3, 7, 11, 4, 8, 12 });
+#if NET7_0_OR_GREATER
+            var res = TypeConvertOperator<int, float, Float32Handler>.Exec(array, new TypeConvertParam()
             {
                 Transpose = true,
                 DimA = 0,
                 DimB = 1
             });
-            _output.WriteLine(convertedArray.Print());
+#else
+            var res = new TypeConvertOperator<int, float, Float32Handler>().Exec(array, new TypeConvertParam()
+            {
+                Transpose = true,
+                DimA = 0,
+                DimB = 1
+            });
+#endif
+            Assert.True(res.IsElementsEqualWith(truth));
+        }
+        [Fact]
+        public void TransposeTest()
+        {
+            var array = NativeArray.FromArray(new NativeLayout(new int[] { 3, 4 }), new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 });
+            var truth = NativeArray.FromArray(new NativeLayout(new int[] { 4, 3 }), new int[] { 1, 5, 9, 2, 6, 10, 3, 7, 11, 4, 8, 12 });
+#if NET7_0_OR_GREATER
+            var res = TransposeOperator<int>.Exec(array, new TransposeParam() { DimA = 0, DimB = 1 });
+#else
+            var res = new TransposeOperator<int>().Exec(array, new TransposeParam(){ DimA = 0, DimB = 1 });
+#endif
+            Assert.True(res.IsElementsEqualWith(truth));
         }
         [Fact]
         public void MatmulTest()
         {
             var a = NativeArray.FromArray(new NativeLayout(new int[] { 3, 4 }), new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 });
             var b = NativeArray.FromArray(new NativeLayout(new int[] { 4, 2 }), new float[] { 1, 2, 1, 2, 1, 1, 2, 3 });
-            var c = new NativeArray<float>(new NativeLayout(new int[] { 3, 2 }), new DefaultNativeMemoryManager());
-            Native.Operators.Naive.MatmulOperator<float>.Exec(a.Span, b.Span, c.Span, a._layout, b._layout, c._layout);
-            _output.WriteLine(c.Print());
+            var truth = NativeArray.FromArray(new NativeLayout(new int[] { 3, 2 }), new float[] { 14, 21, 34, 53, 54, 85 });
+#if NET7_0_OR_GREATER
+            var c = Native.Operators.Naive.MatmulOperator<float>.Exec(a, b);
+#else
+            var c = new Native.Operators.Naive.MatmulOperator<float, Float32Handler>().Exec(a, b);
+#endif
+            Assert.True(c.IsElementsEqualWith(truth));
+        }
+        [Fact]
+        public void BinaryElemWiseTest()
+        {
+            var array = NativeArray.FromArray(new NativeLayout(new int[] { 2, 3 }), new int[] { 1, 2, 3, 4, 5, 6 });
+            var truth = NativeArray.FromArray(new NativeLayout(new int[] { 2, 3 }), new double[] { 2.5, 5.0, 7.5, 10.0, 12.5, 15.0});
+#if NET7_0_OR_GREATER
+            var res = BinaryElemWiseOperator<int, double>.Exec(array, (x, idx) => 2.5 * x);
+#else
+            var res = new BinaryElemWiseOperator<int, double>().Exec(array, (x, idx) => x * 2.5);
+#endif
+            Assert.True(res.IsElementsEqualWith(truth));
+        }
+        [Fact]
+        public void UnaryElemWiseTest()
+        {
+            var array = NativeArray.FromArray(new NativeLayout(new int[] { 2, 3 }), new int[] { 1, 2, 3, 4, 5, 6 });
+            var truth = NativeArray.FromArray(new NativeLayout(new int[] { 2, 3 }), new int[] { 2, 5, 8, 11, 14, 17 });
+#if NET7_0_OR_GREATER
+            UnaryElemWiseOperator<int>.Exec(ref array, (x, idx) => 2 * x + idx);
+#else
+            new UnaryElemWiseOperator<int>().Exec(ref array, (x, idx) => x * 2 + idx);
+#endif
+            Assert.True(array.IsElementsEqualWith(truth));
+        }
+        [Fact]
+        public void TernaryElemWiseTest()
+        {
+            var a = NativeArray.FromArray(new NativeLayout(new int[] { 2, 3 }), new int[] { 1, 2, 3, 4, 5, 6 });
+            var b = NativeArray.FromArray(new NativeLayout(new int[] { 2, 3 }), new float[] { 6, 5, 4, 3, 2, 1 });
+            var truth = NativeArray.FromArray(new NativeLayout(new int[] { 2, 3 }), new double[] { -5, -3, -1, 1, 3, 5});
+#if NET7_0_OR_GREATER
+            var res = TernaryElemWiseOperator<int, float, double>.Exec(a, b, (x, y, idx) => x - y);
+#else
+            var res = new TernaryElemWiseOperator<int, float, double>().Exec(a, b, (x, y, idx) => x - y);
+#endif
+            Assert.True(res.IsElementsEqualWith(truth));
+        }
+        [Fact]
+        public void ReduceSumTest()
+        {
+            var array = NativeArray.FromArray(new NativeLayout(new int[] { 3, 2, 3 }), new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18 });
+            var truth = NativeArray.FromArray(new NativeLayout(new int[] { 3 }), new int[] { 51, 57, 63 });
+#if NET7_0_OR_GREATER
+            var res = ReduceOperator<int, ReduceSumOpHandler<int>>.Exec(array, new ReduceParam(new int[] { 0, 1 }));
+#else
+            var res = new ReduceOperator<int, ReduceSumOpHandler<int, Int32Handler>>().Exec(array, new ReduceParam(new int[] { 0, 1 }));
+#endif
+            _output.WriteLine(res.Print());
+            res._layout.RemoveDanglingAxesInplace();
+            Assert.True(res.IsElementsEqualWith(truth));
+        }
+        [Fact]
+        public void ReduceMaxTest()
+        {
+            var array = NativeArray.FromArray(new NativeLayout(new int[] { 3, 2, 3 }), new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18 });
+            var truth = NativeArray.FromArray(new NativeLayout(new int[] { 3, 2 }), new int[] { 3, 6, 9, 12, 15, 18 });
+#if NET7_0_OR_GREATER
+            var res = ReduceOperator<int, ReduceMaxOpHandler<int>>.Exec(array, new ReduceParam(new int[] { 2 }));
+#else
+            var res = new ReduceOperator<int, ReduceMaxOpHandler<int, Int32Handler>>().Exec(array, new ReduceParam(new int[] { 2 }));
+#endif
+            _output.WriteLine(res.Print());
+            res._layout.RemoveDanglingAxesInplace();
+            Assert.True(res.IsElementsEqualWith(truth));
+        }
+        [Fact]
+        public void ReduceMinTest()
+        {
+            var array = NativeArray.FromArray(new NativeLayout(new int[] { 3, 2, 3, 2 }), 
+                new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 
+                    19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36 }
+            );
+            var truth = NativeArray.FromArray(new NativeLayout(new int[] { 2, 2 }), new int[] { 1, 2, 7, 8 });
+#if NET7_0_OR_GREATER
+            var res = ReduceOperator<int, ReduceMinOpHandler<int>>.Exec(array, new ReduceParam(new int[] { 0, 2 }));
+#else
+            var res = new ReduceOperator<int, ReduceMinOpHandler<int, Int32Handler>>().Exec(array, new ReduceParam(new int[] { 0, 2 }));
+#endif
+            _output.WriteLine(res.Print());
+            res._layout.RemoveDanglingAxesInplace();
+            Assert.True(res.IsElementsEqualWith(truth));
+        }
+
+        [Fact]
+        public void ReduceMeanTest()
+        {
+            var array = NativeArray.FromArray(new NativeLayout(new int[] { 3, 2, 3, 2 }),
+                new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
+                    19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36 }
+            );
+            var truth = NativeArray.FromArray(new NativeLayout(new int[] { 2, 3 }), new int[] { 13, 15, 17, 19, 21, 23 });
+#if NET7_0_OR_GREATER
+            var res = ReduceOperator<int, ReduceMeanOpHandler<int>>.Exec(array, new ReduceParam(new int[] { 0, 3 }));
+#else
+            var res = new ReduceOperator<int, ReduceMeanOpHandler<int, Int32Handler>>().Exec(array, new ReduceParam(new int[] { 0, 3 }));
+#endif
+            _output.WriteLine(res.Print());
+            res._layout.RemoveDanglingAxesInplace();
+            Assert.True(res.IsElementsEqualWith(truth));
         }
     }
 }
\ No newline at end of file
diff --git a/NN.UnitTests/Operators/X86/ArithmeticTest.cs b/NN.UnitTests/Operators/X86/ArithmeticTest.cs
new file mode 100644
index 0000000..432e3cd
--- /dev/null
+++ b/NN.UnitTests/Operators/X86/ArithmeticTest.cs
@@ -0,0 +1,266 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Data;
+using NN.Native.Basic;
+using NN.Native.Basic.DType;
+using System.Runtime.InteropServices;
+using NN.Native.Abstraction.DType;
+using System.Numerics;
+
+namespace NN.UnitTests.Operators.X86
+{
+    public class ArithmeticTest
+    {
+        public (int[], int[])[] Shapes { get; set; } = new (int[], int[])[] {
+                (new int[] {2, 4}, new int[] {2, 4}),
+                (new int[] {7, 16}, new int[] {7, 16}),
+                (new int[] {20, 1}, new int[] {20, 1}),
+                (new int[] {1, 18}, new int[] {1, 18}),
+                (new int[] {99, 128}, new int[] {99, 128}),
+                (new int[] {939, 50}, new int[] {939, 50}),
+            };
+
+        [Fact]
+        public unsafe void TestAddOperatorAllTypes()
+        {
+            GenericTernaryTest<float>(
+#if NET7_0_OR_GREATER
+                (x, y) => NN.Native.Operators.X86.Arithmetic.AddOperator<float>.Exec(x, y),
+#else
+                (x, y) => new NN.Native.Operators.X86.Arithmetic.AddOperator<float, Float32Handler>().Exec(x, y),
+#endif
+                (x, y) => x + y,
+                x => x,
+                -100, 100
+            );
+            GenericTernaryTest<double>(
+#if NET7_0_OR_GREATER
+                (x, y) => NN.Native.Operators.X86.Arithmetic.AddOperator<double>.Exec(x, y),
+#else
+                (x, y) => new NN.Native.Operators.X86.Arithmetic.AddOperator<double, Float64Handler>().Exec(x, y),
+#endif
+                (x, y) => x + y,
+                x => x,
+                -100, 100
+            );
+            GenericTernaryTest<int>(
+#if NET7_0_OR_GREATER
+                (x, y) => NN.Native.Operators.X86.Arithmetic.AddOperator<int>.Exec(x, y),
+#else
+                (x, y) => new NN.Native.Operators.X86.Arithmetic.AddOperator<int, Int32Handler>().Exec(x, y),
+#endif
+                (x, y) => x + y,
+                x => x,
+                -100, 100
+            );
+        }
+
+        [Fact]
+        public unsafe void TestSubtractOperatorAllTypes()
+        {
+            GenericTernaryTest<float>(
+#if NET7_0_OR_GREATER
+                (x, y) => NN.Native.Operators.X86.Arithmetic.SubtractOperator<float>.Exec(x, y),
+#else
+                (x, y) => new NN.Native.Operators.X86.Arithmetic.SubtractOperator<float, Float32Handler>().Exec(x, y),
+#endif
+                (x, y) => x - y,
+                x => x,
+                -100, 100
+            );
+            GenericTernaryTest<double>(
+#if NET7_0_OR_GREATER
+                (x, y) => NN.Native.Operators.X86.Arithmetic.SubtractOperator<double>.Exec(x, y),
+#else
+                (x, y) => new NN.Native.Operators.X86.Arithmetic.SubtractOperator<double, Float64Handler>().Exec(x, y),
+#endif
+                (x, y) => x - y,
+                x => x,
+                -100, 100
+            );
+            GenericTernaryTest<int>(
+#if NET7_0_OR_GREATER
+                (x, y) => NN.Native.Operators.X86.Arithmetic.SubtractOperator<int>.Exec(x, y),
+#else
+                (x, y) => new NN.Native.Operators.X86.Arithmetic.SubtractOperator<int, Int32Handler>().Exec(x, y),
+#endif
+                (x, y) => x - y,
+                x => x,
+                -100, 100
+            );
+        }
+
+        [Fact]
+        public unsafe void TestMultiplyOperatorAllTypes()
+        {
+            GenericTernaryTest<float>(
+#if NET7_0_OR_GREATER
+                (x, y) => NN.Native.Operators.X86.Arithmetic.MultiplyOperator<float>.Exec(x, y),
+#else
+                (x, y) => new NN.Native.Operators.X86.Arithmetic.MultiplyOperator<float, Float32Handler>().Exec(x, y),
+#endif
+                (x, y) => x * y,
+                x => x,
+                -100, 100
+            );
+            GenericTernaryTest<double>(
+#if NET7_0_OR_GREATER
+                (x, y) => NN.Native.Operators.X86.Arithmetic.MultiplyOperator<double>.Exec(x, y),
+#else
+                (x, y) => new NN.Native.Operators.X86.Arithmetic.MultiplyOperator<double, Float64Handler>().Exec(x, y),
+#endif
+                (x, y) => x * y,
+                x => x,
+                -100, 100
+            );
+            GenericTernaryTest<int>(
+#if NET7_0_OR_GREATER
+                (x, y) => NN.Native.Operators.X86.Arithmetic.MultiplyOperator<int>.Exec(x, y),
+#else
+                (x, y) => new NN.Native.Operators.X86.Arithmetic.MultiplyOperator<int, Int32Handler>().Exec(x, y),
+#endif
+                (x, y) => x * y,
+                x => x,
+                -100, 100
+            );
+        }
+
+        [Fact]
+        public unsafe void TestDivideOperatorAllTypes()
+        {
+            GenericTernaryTest<float>(
+#if NET7_0_OR_GREATER
+                (x, y) => NN.Native.Operators.X86.Arithmetic.DivideOperator<float>.Exec(x, y),
+#else
+                (x, y) => new NN.Native.Operators.X86.Arithmetic.DivideOperator<float, Float32Handler>().Exec(x, y),
+#endif
+                (x, y) => x / y,
+                x => x,
+                1, 1000
+            );
+            GenericTernaryTest<double>(
+#if NET7_0_OR_GREATER
+                (x, y) => NN.Native.Operators.X86.Arithmetic.DivideOperator<double>.Exec(x, y),
+#else
+                (x, y) => new NN.Native.Operators.X86.Arithmetic.DivideOperator<double, Float64Handler>().Exec(x, y),
+#endif
+                (x, y) => x / y,
+                x => x,
+                1, 1000
+            );
+            GenericTernaryTest<int>(
+#if NET7_0_OR_GREATER
+                (x, y) => NN.Native.Operators.X86.Arithmetic.DivideOperator<int>.Exec(x, y),
+#else
+                (x, y) => new NN.Native.Operators.X86.Arithmetic.DivideOperator<int, Int32Handler>().Exec(x, y),
+#endif
+                (x, y) => x / y,
+                x => x,
+                1, 1000
+            );
+        }
+
+        [Fact]
+        public unsafe void TestNegativeOperatorAllTypes()
+        {
+            GenericBinaryTest<float>(
+#if NET7_0_OR_GREATER
+                x => NN.Native.Operators.X86.Arithmetic.NegativeOperator<float>.Exec(x),
+#else
+                x => new NN.Native.Operators.X86.Arithmetic.NegativeOperator<float, Float32Handler>().Exec(x),
+#endif
+                x => -x,
+                x => x,
+                1, 1000
+            );
+            GenericBinaryTest<double>(
+#if NET7_0_OR_GREATER
+                x => NN.Native.Operators.X86.Arithmetic.NegativeOperator<double>.Exec(x),
+#else
+                x => new NN.Native.Operators.X86.Arithmetic.NegativeOperator<double, Float64Handler>().Exec(x),
+#endif
+                x => -x,
+                x => x,
+                1, 1000
+            );
+            GenericBinaryTest<int>(
+#if NET7_0_OR_GREATER
+                x => NN.Native.Operators.X86.Arithmetic.NegativeOperator<int>.Exec(x),
+#else
+                x => new NN.Native.Operators.X86.Arithmetic.NegativeOperator<int, Int32Handler>().Exec(x),
+#endif
+                x => -x,
+                x => x,
+                1, 1000
+            );
+        }
+
+        private unsafe void GenericTernaryTest<TData>(
+            Func<NativeArray<TData>, NativeArray<TData>, NativeArray<TData>> x86ExecuteFunc, 
+            Func<TData, TData, TData> ternaryFunc, Func<TData, double> resultConvertFunc, 
+            TData lowerRandomBorder, TData upperRandomBorder, double error = 0.01)
+#if NET7_0_OR_GREATER
+            where TData : unmanaged, INumberBase<TData>
+#else
+            where TData : unmanaged
+#endif
+        {
+            foreach (var shape in Shapes)
+            {
+                var a = NativeArray.Random.Normal<TData>(new NativeLayout(shape.Item1), lowerRandomBorder, upperRandomBorder);
+                var b = NativeArray.Random.Normal<TData>(new NativeLayout(shape.Item2), lowerRandomBorder, upperRandomBorder);
+#if NET7_0_OR_GREATER
+                var naiveResult = Native.Operators.Naive.TernaryElemWiseOperator<TData, TData, TData>.Exec(a, b, (x, y, offset) => ternaryFunc(x, y));
+#else
+                var naiveResult = new Native.Operators.Naive.TernaryElemWiseOperator<TData, TData, TData>().Exec(a, b, (x, y, offset) => ternaryFunc(x, y));
+#endif
+                var x86Result = x86ExecuteFunc(a, b);
+                var enumeratorNaive = NativeLayout.GetIndexEnumerator(naiveResult._layout);
+                var enumeratorX86 = NativeLayout.GetIndexEnumerator(x86Result._layout);
+                var naiveSpan = naiveResult.Span;
+                var x86SPan = x86Result.Span;
+                for (int i = 0; i < naiveResult._layout.TotalElemCount(); i++)
+                {
+                    //Assert.Equal(naiveSpan[enumeratorNaive.MoveNext()], x86SPan[enumeratorX86.MoveNext()]);
+                    var diff = resultConvertFunc(naiveSpan[enumeratorNaive.MoveNext()]) - resultConvertFunc(x86SPan[enumeratorX86.MoveNext()]);
+                    Assert.True(diff > -error && diff < error);
+                }
+            }
+        }
+
+        private unsafe void GenericBinaryTest<TData>(
+            Func<NativeArray<TData>, NativeArray<TData>> x86ExecuteFunc,
+            Func<TData, TData> referenceFunc, Func<TData, double> resultConvertFunc,
+            TData lowerRandomBorder, TData upperRandomBorder, double error = 0.01)
+#if NET7_0_OR_GREATER
+            where TData : unmanaged, INumberBase<TData>
+#else
+            where TData : unmanaged
+#endif
+        {
+            foreach (var shape in Shapes)
+            {
+                var a = NativeArray.Random.Normal<TData>(new NativeLayout(shape.Item1), lowerRandomBorder, upperRandomBorder);
+#if NET7_0_OR_GREATER
+                var naiveResult = Native.Operators.Naive.BinaryElemWiseOperator<TData, TData>.Exec(a, (x, offset) => referenceFunc(x));
+#else
+                var naiveResult = new Native.Operators.Naive.BinaryElemWiseOperator<TData, TData>().Exec(a, (x, offset) => referenceFunc(x));
+#endif
+                var x86Result = x86ExecuteFunc(a);
+                var enumeratorNaive = NativeLayout.GetIndexEnumerator(naiveResult._layout);
+                var enumeratorX86 = NativeLayout.GetIndexEnumerator(x86Result._layout);
+                var naiveSpan = naiveResult.Span;
+                var x86SPan = x86Result.Span;
+                for (int i = 0; i < naiveResult._layout.TotalElemCount(); i++)
+                {
+                    //Assert.Equal(naiveSpan[enumeratorNaive.MoveNext()], x86SPan[enumeratorX86.MoveNext()]);
+                    var diff = resultConvertFunc(naiveSpan[enumeratorNaive.MoveNext()]) - resultConvertFunc(x86SPan[enumeratorX86.MoveNext()]);
+                    Assert.True(diff > -error && diff < error);
+                }
+            }
+        }
+    }
+}
diff --git a/NN.UnitTests/Operators/X86/MatmulTest.cs b/NN.UnitTests/Operators/X86/MatmulTest.cs
index c15453c..6fc9403 100644
--- a/NN.UnitTests/Operators/X86/MatmulTest.cs
+++ b/NN.UnitTests/Operators/X86/MatmulTest.cs
@@ -5,6 +5,10 @@
 using System.Threading.Tasks;
 using NN.Native.Data;
 using NN.Native.Basic;
+using NN.Native.Basic.DType;
+using System.Runtime.InteropServices;
+using NN.Native.Abstraction.DType;
+using System.Numerics;
 
 namespace NN.UnitTests.Operators.X86
 {
@@ -14,28 +18,49 @@ public class MatmulTest
                 (new int[] {2, 4}, new int[] {4, 4}),
                 (new int[] {3, 7}, new int[] {7, 16}),
                 (new int[] {1, 20}, new int[] {20, 1}),
-                (new int[] {10, 1}, new int[] {1, 20}),
-                (new int[] {100, 100}, new int[] {100, 101}),
+                (new int[] {10, 1}, new int[] {1, 18}),
+                (new int[] {16, 99}, new int[] {99, 128}),
+                //(new int[] {1024, 1023}, new int[] {1023, 1026}),
+                (new int[] {20, 28}, new int[] {28, 36}),
+                (new int[] {2, 939}, new int[] {939, 50}),
             };
 
         [Fact]
-        public unsafe void Float32Test()
+        public unsafe void TestAllTypes()
         {
-            foreach(var shape in Shapes)
+            GenericTest<float, Float32Handler>(x => x, x => x);
+            GenericTest<double, Float64Handler>(x => x, x => x);
+            GenericTest<int, Int32Handler>(x => x, x => x);
+        }
+
+        private unsafe void GenericTest<TData, THandler>(Func<int, TData> numberConvertFunc, Func<TData, double> resultConvertFunc)
+#if NET7_0_OR_GREATER
+            where TData: unmanaged, INumberBase<TData>
+#else
+            where TData: unmanaged
+#endif
+            where THandler: INativeDTypeHandler<TData>, new()
+        {
+            foreach (var shape in Shapes)
             {
-                var a = NativeArray.Random.Normal<int>(new NativeLayout(shape.Item1), -10, 10);
-                var b = NativeArray.Random.Normal<int>(new NativeLayout(shape.Item2), -10, 10);
-                var naiveResult = new NativeArray<int>(new NativeLayout(new int[] { shape.Item1[0], shape.Item2[1] }), new DefaultNativeMemoryManager());
-                var x86Result = new NativeArray<int>(new NativeLayout(new int[] { shape.Item1[0], shape.Item2[1] }), new DefaultNativeMemoryManager());
-                Native.Operators.Naive.MatmulOperator<int>.Exec(a.Span, b.Span, naiveResult.Span, a._layout, b._layout, naiveResult._layout);
-                Native.Operators.X86.MatmulOperator<int>.Exec(a.Span, b.Span, x86Result.Span, a._layout, b._layout, x86Result._layout);
+                var a = NativeArray.Random.Normal<TData>(new NativeLayout(shape.Item1), numberConvertFunc(-10), numberConvertFunc(10));
+                var b = NativeArray.Random.Normal<TData>(new NativeLayout(shape.Item2), numberConvertFunc(-10), numberConvertFunc(10));
+#if NET7_0_OR_GREATER
+                var naiveResult = Native.Operators.Naive.MatmulOperator<TData>.Exec(a, b);
+                var x86Result = Native.Operators.X86.MatmulOperator<TData>.Exec(a, b);
+#else
+                var naiveResult = new Native.Operators.Naive.MatmulOperator<TData, THandler>().Exec(a, b);
+                var x86Result = new Native.Operators.X86.MatmulOperator<TData, THandler>().Exec(a, b);
+#endif
                 var enumeratorNaive = NativeLayout.GetIndexEnumerator(naiveResult._layout);
                 var enumeratorX86 = NativeLayout.GetIndexEnumerator(x86Result._layout);
                 var naiveSpan = naiveResult.Span;
                 var x86SPan = x86Result.Span;
-                for(int i = 0; i < naiveResult._layout.TotalElemCount(); i++)
+                for (int i = 0; i < naiveResult._layout.TotalElemCount(); i++)
                 {
-                    Assert.Equal(naiveSpan[enumeratorNaive.MoveNext()], x86SPan[enumeratorX86.MoveNext()]);
+                    //Assert.Equal(naiveSpan[enumeratorNaive.MoveNext()], x86SPan[enumeratorX86.MoveNext()]);
+                    var diff = resultConvertFunc(naiveSpan[enumeratorNaive.MoveNext()]) - resultConvertFunc(x86SPan[enumeratorX86.MoveNext()]);
+                    Assert.True(diff > -0.01 && diff < 0.01);
                 }
             }
         }
diff --git a/NN.sln b/NN.sln
index e3b9761..63faa76 100644
--- a/NN.sln
+++ b/NN.sln
@@ -13,7 +13,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NN.BenchmarkTest", "NN.Benc
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NN.Core", "NN.Core\NN.Core.csproj", "{C61FC3B6-FD8C-4BDA-BC69-AB4BD46C8867}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmark", "Benchmark\Benchmark.csproj", "{9A84C621-DF8A-4D37-B980-F8DCC4638B40}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LinearRegression", "Benchmark\LinearRegression.csproj", "{9A84C621-DF8A-4D37-B980-F8DCC4638B40}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
diff --git a/NN/Abstraction/Common/Params.cs b/NN/Abstraction/Common/Params.cs
new file mode 100644
index 0000000..be54378
--- /dev/null
+++ b/NN/Abstraction/Common/Params.cs
@@ -0,0 +1,34 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.Common
+{
+    public readonly record struct MatmulParam(
+        bool TransposeA,
+        bool TransposeB
+    );
+    public readonly record struct TypeConvertParam(
+        bool Transpose,
+        int DimA,
+        int DimB
+    );
+    public readonly record struct TransposeParam(
+        int DimA,
+        int DimB
+    );
+    public readonly ref struct ReduceParam
+    {
+        public readonly ReadOnlySpan<int> Dims;
+        public ReduceParam(ReadOnlySpan<int> dims)
+        {
+            Dims = dims;
+        }
+        public ReduceParam(params int[] dims)
+        {
+            Dims = dims.AsSpan();
+        }
+    }
+}
diff --git a/NN/Abstraction/DType/IBinaryDTypeHandler.cs b/NN/Abstraction/DType/IBinaryDTypeHandler.cs
index a6cb9c6..a1e7cd4 100644
--- a/NN/Abstraction/DType/IBinaryDTypeHandler.cs
+++ b/NN/Abstraction/DType/IBinaryDTypeHandler.cs
@@ -23,5 +23,7 @@ public unsafe interface IBinaryDTypeHandler<T> where T : unmanaged
         /// <param name="b"></param>
         /// <returns></returns>
         bool IsLess(T a, T b);
+
+        T GetNegative(T x);
     }
 }
diff --git a/NN/Abstraction/DType/ITernaryDTypeHandler.cs b/NN/Abstraction/DType/ITernaryDTypeHandler.cs
index 488e407..13123f8 100644
--- a/NN/Abstraction/DType/ITernaryDTypeHandler.cs
+++ b/NN/Abstraction/DType/ITernaryDTypeHandler.cs
@@ -20,7 +20,7 @@ public interface ITernaryDTypeHandler<T> where T : unmanaged
         /// </summary>
         /// <param name="a"></param>
         /// <param name="b"></param>
-        T Substract(T a, T b);
+        T Subtract(T a, T b);
         /// <summary>
         /// a * b
         /// </summary>
diff --git a/NN/Abstraction/Data/INativeMemoryManager.cs b/NN/Abstraction/Data/INativeMemoryManager.cs
index 20fcd55..8aea5a4 100644
--- a/NN/Abstraction/Data/INativeMemoryManager.cs
+++ b/NN/Abstraction/Data/INativeMemoryManager.cs
@@ -3,11 +3,14 @@
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using NN.Native.Data;
 
 namespace NN.Native.Abstraction.Data
 {
     public interface INativeMemoryManager
     {
+        private static readonly INativeMemoryManager _defaultMemoryManager = new DefaultNativeMemoryManager();
+        public static INativeMemoryManager Default { get => _defaultMemoryManager; }
         Memory<T> AllocateMemory<T>(int elems) where T: unmanaged;
     }
 }
diff --git a/NN/Abstraction/Operators/Arithmetic/IAddOperator.cs b/NN/Abstraction/Operators/Arithmetic/IAddOperator.cs
new file mode 100644
index 0000000..c814b1b
--- /dev/null
+++ b/NN/Abstraction/Operators/Arithmetic/IAddOperator.cs
@@ -0,0 +1,32 @@
+﻿using NN.Native.Abstraction.Data;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Exceptions;
+
+namespace NN.Native.Abstraction.Operators.Arithmetic
+{
+    public interface IAddOperator<T>: ITernaryOperator where T: unmanaged
+    {
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(in NativeArray<T> a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null);
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(in NativeArray<T> a, T b, INativeMemoryManager? memoryManager = null);
+
+        public static NativeLayout DeduceLayout(in NativeLayout a, in NativeLayout b)
+        {
+            if (!a.IsSameShape(b))
+            {
+                throw new InvalidShapeException();
+            }
+            return NativeLayout.ShapeLike(a);
+        }
+        public static NativeLayout DeduceLayout(in NativeLayout a)
+        {
+            return NativeLayout.ShapeLike(a);
+        }
+    }
+}
diff --git a/NN/Abstraction/Operators/Arithmetic/IDivideOperator.cs b/NN/Abstraction/Operators/Arithmetic/IDivideOperator.cs
new file mode 100644
index 0000000..70466d2
--- /dev/null
+++ b/NN/Abstraction/Operators/Arithmetic/IDivideOperator.cs
@@ -0,0 +1,36 @@
+﻿using NN.Native.Abstraction.Data;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Exceptions;
+
+namespace NN.Native.Abstraction.Operators.Arithmetic
+{
+    public interface IDivideOperator<T> : ITernaryOperator where T : unmanaged
+    {
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(in NativeArray<T> a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null);
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(in NativeArray<T> a, T b, INativeMemoryManager? memoryManager = null);
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(T a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null);
+
+        public static NativeLayout DeduceLayout(in NativeLayout a, in NativeLayout b)
+        {
+            if (!a.IsSameShape(b))
+            {
+                throw new InvalidShapeException();
+            }
+            return NativeLayout.ShapeLike(a);
+        }
+        public static NativeLayout DeduceLayout(in NativeLayout a)
+        {
+            return NativeLayout.ShapeLike(a);
+        }
+    }
+}
diff --git a/NN/Abstraction/Operators/Arithmetic/IMultiplyOperator.cs b/NN/Abstraction/Operators/Arithmetic/IMultiplyOperator.cs
new file mode 100644
index 0000000..4534d47
--- /dev/null
+++ b/NN/Abstraction/Operators/Arithmetic/IMultiplyOperator.cs
@@ -0,0 +1,32 @@
+﻿using NN.Native.Abstraction.Data;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Exceptions;
+
+namespace NN.Native.Abstraction.Operators.Arithmetic
+{
+    public interface IMultiplyOperator<T> : ITernaryOperator where T : unmanaged
+    {
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(in NativeArray<T> a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null);
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(in NativeArray<T> a, T b, INativeMemoryManager? memoryManager = null);
+
+        public static NativeLayout DeduceLayout(in NativeLayout a, in NativeLayout b)
+        {
+            if (!a.IsSameShape(b))
+            {
+                throw new InvalidShapeException();
+            }
+            return NativeLayout.ShapeLike(a);
+        }
+        public static NativeLayout DeduceLayout(in NativeLayout a)
+        {
+            return NativeLayout.ShapeLike(a);
+        }
+    }
+}
diff --git a/NN/Abstraction/Operators/Arithmetic/INegativeOperator.cs b/NN/Abstraction/Operators/Arithmetic/INegativeOperator.cs
new file mode 100644
index 0000000..7c70e9c
--- /dev/null
+++ b/NN/Abstraction/Operators/Arithmetic/INegativeOperator.cs
@@ -0,0 +1,25 @@
+﻿using NN.Native.Abstraction.Data;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Exceptions;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.Operators.Arithmetic
+{
+    public interface INegativeOperator<T> : ITernaryOperator where T : unmanaged
+    {
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(in NativeArray<T> src, INativeMemoryManager? memoryManager = null);
+
+        public static NativeLayout DeduceLayout(in NativeLayout src)
+        {
+            return NativeLayout.ShapeLike(src);
+        }
+    }
+}
diff --git a/NN/Abstraction/Operators/Arithmetic/ISubtractOperator.cs b/NN/Abstraction/Operators/Arithmetic/ISubtractOperator.cs
new file mode 100644
index 0000000..4196f6c
--- /dev/null
+++ b/NN/Abstraction/Operators/Arithmetic/ISubtractOperator.cs
@@ -0,0 +1,36 @@
+﻿using NN.Native.Abstraction.Data;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Exceptions;
+
+namespace NN.Native.Abstraction.Operators.Arithmetic
+{
+    public interface ISubtractOperator<T> : ITernaryOperator where T : unmanaged
+    {
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(in NativeArray<T> a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null);
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(in NativeArray<T> a, T b, INativeMemoryManager? memoryManager = null);
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(T a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null);
+
+        public static NativeLayout DeduceLayout(in NativeLayout a, in NativeLayout b)
+        {
+            if (!a.IsSameShape(b))
+            {
+                throw new InvalidShapeException();
+            }
+            return NativeLayout.ShapeLike(a);
+        }
+        public static NativeLayout DeduceLayout(in NativeLayout a)
+        {
+            return NativeLayout.ShapeLike(a);
+        }
+    }
+}
diff --git a/NN/Abstraction/Operators/IBinaryElemWiseOperator.cs b/NN/Abstraction/Operators/IBinaryElemWiseOperator.cs
new file mode 100644
index 0000000..150a78d
--- /dev/null
+++ b/NN/Abstraction/Operators/IBinaryElemWiseOperator.cs
@@ -0,0 +1,25 @@
+﻿using NN.Native.Data;
+using NN.Native.Abstraction.Common;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Data;
+using NN.Native.Basic;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public interface IBinaryElemWiseOperator<TA, TB>: IBinaryOperator where TA: unmanaged where TB : unmanaged
+    {
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<TB> Exec(in NativeArray<TA> src, Func<TA, int, TB> elemFunc, INativeMemoryManager? memoryManager = null);
+
+        internal static NativeLayout DeduceLayout(in NativeLayout layoutSrc)
+        {
+            return NativeLayout.ShapeLike(layoutSrc);
+        }
+    }
+}
diff --git a/NN/Abstraction/Operators/IDotOperator.cs b/NN/Abstraction/Operators/IDotOperator.cs
new file mode 100644
index 0000000..309f9d3
--- /dev/null
+++ b/NN/Abstraction/Operators/IDotOperator.cs
@@ -0,0 +1,31 @@
+﻿using NN.Native.Abstraction.Data;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Exceptions;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public unsafe interface IDotOperator<T> : ITernaryOperator where T : unmanaged
+    {
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(in NativeArray<T> a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null);
+
+        internal static NativeLayout DeduceLayout(ref NativeLayout a, ref NativeLayout b)
+        {
+            //TODO: broadcast
+            if (!a.IsSameShape(b))
+            {
+                throw new InvalidShapeException();
+            }
+            return NativeLayout.ShapeLike(a);
+        }
+    }
+}
diff --git a/NN/Abstraction/Operators/IMatmulOperator.cs b/NN/Abstraction/Operators/IMatmulOperator.cs
index 4e5abe7..87857f1 100644
--- a/NN/Abstraction/Operators/IMatmulOperator.cs
+++ b/NN/Abstraction/Operators/IMatmulOperator.cs
@@ -1,13 +1,69 @@
-﻿using NN.Native.Basic;
+﻿using NN.Native.Abstraction.Common;
+using NN.Native.Abstraction.Data;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Exceptions;
+using System.Diagnostics;
 
 namespace NN.Native.Abstraction.Operators
 {
     public unsafe interface IMatmulOperator<T>: ITernaryOperator where T : unmanaged
     {
 #if NET7_0_OR_GREATER
-        static abstract void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
-#else
-        void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout LayoutA, in NativeLayout LayoutB, in NativeLayout LayoutC);
+        static abstract
 #endif
+        NativeArray<T> Exec(in NativeArray<T> a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null);
+
+        internal static NativeLayout DeduceLayout(ref NativeLayout a, ref NativeLayout b)
+        {
+            Debug.Assert(a.Ndim <= 2 && b.Ndim <= 2);
+            if (a.Ndim <= 0 || b.Ndim <= 0)
+            {
+                throw new InvalidShapeException();
+            }
+            if (a.Ndim == 1 && b.Ndim == 1)
+            {
+                if (a.Shape[0] == 1 && b.Shape[0] == 1)
+                {
+                    return new NativeLayout(1);
+                }
+                throw new InvalidShapeException();
+            }
+            else if (a.Ndim == 2 && b.Ndim == 1)
+            {
+                if (a.Shape[1] == b.Shape[0])
+                {
+                    b.AddAxisInplace(1);
+                    return new NativeLayout(a.Shape[0], b.Shape[1]);
+                }
+                else
+                {
+                    throw new InvalidShapeException("Shape mismatched for matmul.");
+                }
+            }
+            else if (a.Ndim == 1 && b.Ndim == 2)
+            {
+                if (a.Shape[0] == b.Shape[0])
+                {
+                    a.AddAxisInplace(0);
+                    return new NativeLayout(a.Shape[0], b.Shape[1]);
+                }
+                else
+                {
+                    throw new InvalidShapeException("Shape mismatched for matmul.");
+                }
+            }
+            else
+            {
+                if (a.Shape[1] == b.Shape[0])
+                {
+                    return new NativeLayout(a.Shape[0], b.Shape[1]);
+                }
+                else
+                {
+                    throw new InvalidShapeException("Shape mismatched for matmul.");
+                }
+            }
+        }
     }
 }
diff --git a/NN/Abstraction/Operators/IOperator.cs b/NN/Abstraction/Operators/IOperator.cs
index c2fa746..80077d1 100644
--- a/NN/Abstraction/Operators/IOperator.cs
+++ b/NN/Abstraction/Operators/IOperator.cs
@@ -10,9 +10,11 @@ public interface IOperator
     {
 #if NET7_0_OR_GREATER
         static abstract bool IsThreadSafe { get; }
+        static abstract bool RequireContiguousArray { get; }
         static abstract OperatorHandlerType HandlerType { get; }
 #else
         bool IsThreadSafe { get; }
+        bool RequireContiguousArray { get; }
         OperatorHandlerType HandlerType { get; }
 #endif
     }
diff --git a/NN/Abstraction/Operators/IReduceOperator.cs b/NN/Abstraction/Operators/IReduceOperator.cs
new file mode 100644
index 0000000..fb64b1a
--- /dev/null
+++ b/NN/Abstraction/Operators/IReduceOperator.cs
@@ -0,0 +1,166 @@
+﻿using NN.Native.Abstraction.Common;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Abstraction.DType;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Data;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public interface IReduceOperator<T, TReduceHandler> : IBinaryOperator where T : unmanaged where TReduceHandler : IReduceOpHandler<T>
+    {
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(in NativeArray<T> src, in ReduceParam param, INativeMemoryManager? memoryManager = null);
+
+        public static NativeLayout DeduceLayout(in NativeLayout src, in ReduceParam param)
+        {
+            int[] shape = new int[src._ndim];
+            var span = shape.AsSpan();
+            src._shape.AsSpan(0, src._ndim).CopyTo(span);
+            for(int i = 0; i < param.Dims.Length; i++)
+            {
+                span[param.Dims[i]] = 1;
+            }
+            return new NativeLayout(shape);
+        }
+    }
+    public interface IReduceOpHandler<T> where T: unmanaged
+    {
+#if NET7_0_OR_GREATER
+        static abstract T GetValue(T x);
+        static abstract T Apply(T x, T y);
+        static abstract T SetValue(T x, T size);
+#else
+        T GetValue(T x);
+        T Apply(T x, T y);
+        T SetValue(T x, T size);
+#endif
+    }
+
+    #region ReduceOpHandler Implementations
+#if NET7_0_OR_GREATER
+    public class ReduceMaxOpHandler<TData> : IReduceOpHandler<TData> where TData : unmanaged, IComparisonOperators<TData, TData>
+    {
+        static
+#else
+    public class ReduceMaxOpHandler<TData, TDataHandler> : IReduceOpHandler<TData> where TData : unmanaged where TDataHandler: INativeDTypeHandler<TData>, new()
+    {
+        private static TDataHandler _handler = new TDataHandler();
+#endif
+        public TData GetValue(TData x) => x;
+
+#if NET7_0_OR_GREATER
+        public static TData Apply(TData x, TData y)
+        {
+            return x > y ? x : y;
+        }
+#else
+        public TData Apply(TData x, TData y)
+        {
+            return _handler.IsGraeter(x, y) ? x : y;
+        }
+#endif
+
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public TData SetValue(TData x, TData _) => x;
+    }
+
+#if NET7_0_OR_GREATER
+    public class ReduceMinOpHandler<TData> : IReduceOpHandler<TData> where TData : unmanaged, IComparisonOperators<TData, TData>
+    {
+        static
+#else
+    public class ReduceMinOpHandler<TData, TDataHandler> : IReduceOpHandler<TData> where TData : unmanaged where TDataHandler: INativeDTypeHandler<TData>, new()
+    {
+        private static TDataHandler _handler = new TDataHandler();
+#endif
+        public TData GetValue(TData x) => x;
+
+#if NET7_0_OR_GREATER
+        public static TData Apply(TData x, TData y)
+        {
+            return x < y ? x : y;
+        }
+#else
+        public TData Apply(TData x, TData y)
+        {
+            return _handler.IsLess(x, y) ? x : y;
+        }
+#endif
+
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public TData SetValue(TData x, TData _) => x;
+    }
+
+#if NET7_0_OR_GREATER
+    public class ReduceMeanOpHandler<TData> : IReduceOpHandler<TData> where TData : unmanaged, IAdditionOperators<TData, TData, TData>, IDivisionOperators<TData, TData, TData>
+    {
+        static
+#else
+    public class ReduceMeanOpHandler<TData, TDataHandler> : IReduceOpHandler<TData> where TData : unmanaged where TDataHandler: INativeDTypeHandler<TData>, new()
+    {
+        private static TDataHandler _handler = new TDataHandler();
+#endif
+        public TData GetValue(TData x) => x;
+
+#if NET7_0_OR_GREATER
+        public static TData Apply(TData x, TData y)
+        {
+            return x + y;
+        }
+#else
+        public TData Apply(TData x, TData y)
+        {
+            return _handler.Add(x, y);
+        }
+#endif
+
+#if NET7_0_OR_GREATER
+        public static TData SetValue(TData x, TData size) => x / size;
+#else
+        public TData SetValue(TData x, TData size) => _handler.Divide(x, size);
+#endif
+    }
+
+#if NET7_0_OR_GREATER
+    public class ReduceSumOpHandler<TData> : IReduceOpHandler<TData> where TData : unmanaged, IAdditionOperators<TData, TData, TData>
+    {
+        static
+#else
+    public class ReduceSumOpHandler<TData, TDataHandler> : IReduceOpHandler<TData> where TData : unmanaged where TDataHandler: INativeDTypeHandler<TData>, new()
+    {
+        private static TDataHandler _handler = new TDataHandler();
+#endif
+        public TData GetValue(TData x) => x;
+
+#if NET7_0_OR_GREATER
+        public static TData Apply(TData x, TData y)
+        {
+            return x + y;
+        }
+#else
+        public TData Apply(TData x, TData y)
+        {
+            return _handler.Add(x, y);
+        }
+#endif
+
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public TData SetValue(TData x, TData _) => x;
+    }
+
+    #endregion
+}
diff --git a/NN/Abstraction/Operators/ITernaryElemWiseOperator.cs b/NN/Abstraction/Operators/ITernaryElemWiseOperator.cs
new file mode 100644
index 0000000..89dca65
--- /dev/null
+++ b/NN/Abstraction/Operators/ITernaryElemWiseOperator.cs
@@ -0,0 +1,41 @@
+﻿using NN.Native.Abstraction.Data;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Exceptions;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public interface ITernaryElemWiseOperator<TA, TB, TC>: ITernaryOperator where TA: unmanaged where TB: unmanaged where TC : unmanaged
+    {
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<TC> Exec(in NativeArray<TA> a, in NativeArray<TB> b, Func<TA, TB, int, TC> elemFunc, INativeMemoryManager? memoryManager = null);
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<TC> Exec(in NativeArray<TA> a, TB b, Func<TA, TB, int, TC> elemFunc, INativeMemoryManager? memoryManager = null);
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<TC> Exec(TA a, in NativeArray<TB> b, Func<TA, TB, int, TC> elemFunc, INativeMemoryManager? memoryManager = null);
+
+        public static NativeLayout DeduceLayout(in NativeLayout a, in NativeLayout b)
+        {
+            if (!a.IsSameShape(b))
+            {
+                throw new InvalidShapeException();
+            }
+            return NativeLayout.ShapeLike(a);
+        }
+        public static NativeLayout DeduceLayout(in NativeLayout a)
+        {
+            return NativeLayout.ShapeLike(a);
+        }
+    }
+}
diff --git a/NN/Abstraction/Operators/ITransposeOperator.cs b/NN/Abstraction/Operators/ITransposeOperator.cs
new file mode 100644
index 0000000..f4e3314
--- /dev/null
+++ b/NN/Abstraction/Operators/ITransposeOperator.cs
@@ -0,0 +1,30 @@
+﻿using NN.Native.Abstraction.Common;
+using NN.Native.Abstraction.Data;
+using NN.Native.Basic;
+using NN.Native.Data;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public interface ITransposeOperator<T>: IBinaryOperator where T: unmanaged
+    {
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        NativeArray<T> Exec(in NativeArray<T> src, in TransposeParam param, INativeMemoryManager? memoryManager = null);
+
+        public static NativeLayout DeduceLayout(in NativeLayout src, in TransposeParam param)
+        {
+            int[] shape = new int[src._ndim];
+            src.Shape.AsSpan(0, src._ndim).CopyTo(shape);
+            var temp = shape[param.DimA];
+            shape[param.DimA] = shape[param.DimB];
+            shape[param.DimB] = temp;
+            return new NativeLayout(shape);
+        }
+    }
+}
diff --git a/NN/Abstraction/Operators/ITypeConvertOperator.cs b/NN/Abstraction/Operators/ITypeConvertOperator.cs
index ef39c37..763a125 100644
--- a/NN/Abstraction/Operators/ITypeConvertOperator.cs
+++ b/NN/Abstraction/Operators/ITypeConvertOperator.cs
@@ -1,15 +1,30 @@
 ﻿
+using NN.Native.Abstraction.Common;
+using NN.Native.Abstraction.Data;
 using NN.Native.Basic;
-using NN.Native.Operators.Common.Params;
+using NN.Native.Data;
 
 namespace NN.Native.Abstraction.Operators
 {
     public interface ITypeConvertOperator<TA, TB>: IBinaryOperator where TA: unmanaged where TB : unmanaged
     {
 #if NET7_0_OR_GREATER
-        static abstract void Exec(ReadOnlySpan<TA> a, Span<TB> b, in NativeLayout layoutA, in NativeLayout layoutB, in TypeConvertParam param);
-#else
-        void Exec(ReadOnlySpan<TA> a, Span<TB> b, in NativeLayout layoutA, in NativeLayout layoutB, in TypeConvertParam param);
+        static abstract
 #endif
+        NativeArray<TB> Exec(in NativeArray<TA> src, in TypeConvertParam param, INativeMemoryManager? memoryManager = null);
+
+        public static NativeLayout DeduceLayout(in NativeLayout src, in TypeConvertParam param)
+        {
+            if (!param.Transpose)
+            {
+                return NativeLayout.ShapeLike(src);
+            }
+            int[] shape = new int[src._ndim];
+            src._shape.AsSpan(0, src._ndim).CopyTo(shape);
+            var temp = shape[param.DimA];
+            shape[param.DimA] = shape[param.DimB];
+            shape[param.DimB] = temp;
+            return new NativeLayout(shape);
+        }
     }
 }
diff --git a/NN/Abstraction/Operators/IUnaryElemWiseOperator.cs b/NN/Abstraction/Operators/IUnaryElemWiseOperator.cs
new file mode 100644
index 0000000..2d2d709
--- /dev/null
+++ b/NN/Abstraction/Operators/IUnaryElemWiseOperator.cs
@@ -0,0 +1,18 @@
+﻿using NN.Native.Abstraction.Data;
+using NN.Native.Data;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Abstraction.Operators
+{
+    public interface IUnaryElemWiseOperator<T>: IUnaryOperator where T: unmanaged
+    {
+#if NET7_0_OR_GREATER
+        static abstract
+#endif
+        void Exec(ref NativeArray<T> src, Func<T, int, T> elemFunc);
+    }
+}
diff --git a/NN/Basic/DType/FloatHandler.cs b/NN/Basic/DType/Float32Handler.cs
similarity index 53%
rename from NN/Basic/DType/FloatHandler.cs
rename to NN/Basic/DType/Float32Handler.cs
index 3c36548..735bd4d 100644
--- a/NN/Basic/DType/FloatHandler.cs
+++ b/NN/Basic/DType/Float32Handler.cs
@@ -12,28 +12,45 @@ public class Float32Handler: INativeDTypeHandler<float>,
         #endregion
 
         #region binary
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
         public bool AreEqual(float a, float b) => a == b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
         public bool IsGraeter(float a, float b) => a > b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
         public bool IsLess(float a, float b) => a < b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public float GetNegative(float x) => -x;
         #endregion
 
         #region ternary
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
         public float Add(float a, float b) => a + b;
-        public float Substract(float a, float b) => a - b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public float Subtract(float a, float b) => a - b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
         public float Multiply(float a, float b) => a * b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
         public float Divide(float a, float b) => a / b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
         public float Module(float a, float b) => a % b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
         public float MultiplyAndAdd(float a, float b, float dst) => dst + a * b;
         #endregion
 
         #region convert
 #if NET7_0_OR_GREATER
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static float Convert(float a) => a;
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static float Convert(int a) => a;
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static float Convert(bool a) => a ? .1f : .0f;
 #else
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
         public float Convert(float a) => a;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
         public float Convert(int a) => a;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
         public float Convert(bool a) => a ? .1f : .0f;
 #endif
         #endregion
diff --git a/NN/Basic/DType/Float64Handler.cs b/NN/Basic/DType/Float64Handler.cs
new file mode 100644
index 0000000..a63a2a5
--- /dev/null
+++ b/NN/Basic/DType/Float64Handler.cs
@@ -0,0 +1,67 @@
+﻿using NN.Native.Abstraction.DType;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Basic.DType
+{
+    public class Float64Handler : INativeDTypeHandler<double>,
+            INativeConvertible<int, double>, INativeConvertible<bool, double>, INativeConvertible<float, double>, INativeConvertible<double, double>
+    {
+        #region unary
+        public double Zero { get => .0; }
+        public double One { get => 1.0; }
+        #endregion
+
+        #region binary
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool AreEqual(double a, double b) => a == b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool IsGraeter(double a, double b) => a > b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool IsLess(double a, double b) => a < b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public double GetNegative(double x) => -x;
+        #endregion
+
+        #region ternary
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public double Add(double a, double b) => a + b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public double Subtract(double a, double b) => a - b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public double Multiply(double a, double b) => a * b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public double Divide(double a, double b) => a / b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public double Module(double a, double b) => a % b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public double MultiplyAndAdd(double a, double b, double dst) => dst + a * b;
+        #endregion
+
+        #region convert
+#if NET7_0_OR_GREATER
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static double Convert(double a) => a;
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static double Convert(int a) => a;
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static double Convert(float a) => a;
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static double Convert(bool a) => a ? .1f : .0f;
+#else
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public double Convert(double a) => a;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public double Convert(int a) => a;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public double Convert(float a) => a;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public double Convert(bool a) => a ? .1f : .0f;
+#endif
+        #endregion
+    }
+}
diff --git a/NN/Basic/DType/Int32Handler.cs b/NN/Basic/DType/Int32Handler.cs
index 28b8744..55717b2 100644
--- a/NN/Basic/DType/Int32Handler.cs
+++ b/NN/Basic/DType/Int32Handler.cs
@@ -1,39 +1,57 @@
-﻿//using NN.Native.Abstraction.DType;
+﻿using NN.Native.Abstraction.DType;
+using System.Runtime.CompilerServices;
 
-//namespace NN.Native.Basic.DType
-//{
-//    public class Int32Handler : IUnaryDTypeHandler<int>, IBinaryDTypeHandler<int, int>, ITernaryDTypeHandler<int, int, int>, INativeConvertible<int, int>, INativeConvertible<bool, int>
-//    {
-//        #region unary
-//        public static int Zero { get => 0; }
-//        public static int One { get => 1; }
-//        #endregion
+namespace NN.Native.Basic.DType
+{
+    public class Int32Handler : INativeDTypeHandler<int>,INativeConvertible<int, int>, INativeConvertible<bool, int>, INativeConvertible<float, int>
+    {
+        #region unary
+        public int Zero { get => 0; }
+        public int One { get => 1; }
+        #endregion
 
-//        #region binary
-//        public static void AddInplace(ref int src, in int addend) => src += addend;
-//        public static void SubstractInplace(ref int src, in int subtrahend) => src -= subtrahend;
-//        public static void MultiplyInplace(ref int src, in int multiplicator) => src *= multiplicator;
-//        public static void DivideInplace(ref int src, in int divisor) => src /= divisor;
-//        public static void ModuleInplace(ref int src, in int modulus) => src %= modulus;
-//        public static void LeftShift(in int src, ref int dst, int offset) => dst = src << offset;
-//        public static void RightShift(in int src, ref int dst, int offset) => dst = src >> offset;
-//        public static bool AreEqual(in int a, in int b) => a == b;
-//        public static bool IsGraeter(in int a, in int b) => a > b;
-//        public static bool IsLess(in int a, in int b) => a < b;
-//        #endregion
+        #region binary
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool AreEqual(int a, int b) => a == b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool IsGraeter(int a, int b) => a > b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool IsLess(int a, int b) => a < b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int GetNegative(int x) => -x;
+        #endregion
 
-//        #region ternary
-//        public static void Add(in int a, in int b, ref int dst) => dst = a + b;
-//        public static void Substract(in int a, in int b, ref int dst) => dst = a - b;
-//        public static void Multiply(in int a, in int b, ref int dst) => dst = a * b;
-//        public static void Divide(in int a, in int b, ref int dst) => dst = a / b;
-//        public static void Module(in int a, in int b, ref int dst) => dst = a % b;
-//        public static void MultiplyAndAdd(in int a, in int b, ref int dst) => dst += a * b;
-//        #endregion
+        #region ternary
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int Add(int a, int b) => a + b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int Subtract(int a, int b) => a - b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int Multiply(int a, int b) => a * b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int Divide(int a, int b) => a / b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int Module(int a, int b) => a % b;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int MultiplyAndAdd(int a, int b, int dst) => dst + a * b;
+        #endregion
 
-//        #region convert
-//        public static void Convert(in int a, ref int b) => b = a;
-//        public static void Convert(in bool a, ref int b) => b = a ? 1 : 0;
-//        #endregion
-//    }
-//}
+        #region convert
+#if NET7_0_OR_GREATER
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int Convert(int a) => a;
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int Convert(float a) => (int)a;
+        [method:MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int Convert(bool a) => a ? 1 : 0;
+#else
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int Convert(int a) => a;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int Convert(float a) => (int)a;
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int Convert(bool a) => a ? 1 : 0;
+#endif
+        #endregion
+    }
+}
diff --git a/NN/Basic/NativeLayout.cs b/NN/Basic/NativeLayout.cs
index f2eb6fb..ecfe987 100644
--- a/NN/Basic/NativeLayout.cs
+++ b/NN/Basic/NativeLayout.cs
@@ -1,4 +1,5 @@
 ﻿using NN.Native.Abstraction;
+using NN.Native.Exceptions;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
@@ -19,6 +20,10 @@ public struct NativeLayout
         internal int[] _shape;
         internal int[] _stride;
 
+        public int Ndim { get => _ndim; }
+        public int[] Shape { get => _shape; }
+        public int[] Stride { get => _stride; }
+
         #region constructor
         public NativeLayout()
         {
@@ -26,10 +31,15 @@ public NativeLayout()
             _shape = new int[MAX_NDIM];
             _stride = new int[MAX_NDIM];
         }
-        public NativeLayout(int[] shape)
+        public NativeLayout(params int[] shape)
         {
-            _ndim = shape.Length;
-            Debug.Assert(_ndim > 0 && _ndim < MAX_NDIM);
+            _ndim = 0;
+            for(int i = 0; i < shape.Length; i++)
+            {
+                if (shape[i] != 0) _ndim++;
+                else if (i > 0 && i < shape.Length - 1) throw new InvalidShapeException();
+            }
+            Debug.Assert(_ndim > 0 && _ndim <= MAX_NDIM);
             _shape = new int[MAX_NDIM];
             _stride = new int[MAX_NDIM];
             shape.CopyTo(_shape, 0);
@@ -41,6 +51,21 @@ public NativeLayout(int[] shape)
                 s *= shape[_ndim - i - 1];
             }
         }
+        public NativeLayout(ReadOnlySpan<int> shape)
+        {
+            _ndim = shape.Length;
+            Debug.Assert(_ndim > 0 && _ndim <= MAX_NDIM);
+            _shape = new int[MAX_NDIM];
+            _stride = new int[MAX_NDIM];
+            shape.CopyTo(_shape);
+            // init contiguous stride
+            int s = 1;
+            for (int i = 0; i < _ndim; i++)
+            {
+                _stride[_ndim - i - 1] = s;
+                s *= shape[_ndim - i - 1];
+            }
+        }
         public NativeLayout(int[] shape, int[] stride)
         {
             _ndim = shape.Length;
@@ -50,7 +75,7 @@ public NativeLayout(int[] shape, int[] stride)
             shape.CopyTo(_shape, 0);
             stride.CopyTo(_stride, 0);
         }
-        public NativeLayout(ref NativeLayout layout)
+        public NativeLayout(in NativeLayout layout)
         {
             _ndim = layout._ndim;
             Debug.Assert(_ndim > 0 && _ndim < MAX_NDIM);
@@ -92,14 +117,18 @@ public readonly int TotalElemCount()
             for (int i = 0; i < _ndim; i++) res *= _shape[i];
             return res;
         }
-        public readonly bool IsSameShape(ref NativeLayout other)
+        public readonly bool IsSameShape(in NativeLayout other)
         {
             if (_ndim != other._ndim) return false;
-            return _shape.AsSpan().SequenceEqual(other._shape.AsSpan());
+            for(int i = 0; i < _ndim; i++)
+            {
+                if (_shape[i] != other._shape[i]) return false;
+            }
+            return true;
         }
-        public readonly bool IsSameLayout(ref NativeLayout other)
+        public readonly bool IsSameLayout(in NativeLayout other)
         {
-            return IsSameShape(ref other) && _stride.AsSpan().SequenceEqual(other._stride);
+            return IsSameShape(other) && _stride.AsSpan().SequenceEqual(other._stride);
         }
         #endregion
 
@@ -129,6 +158,121 @@ public void AddAxisInplace(int axis, int shape, int stride)
 
         public void AddAxisInplace(int axis) => AddAxisInplace(axis, 1, _stride[axis] * _shape[axis]);
 
+        public void BroadcastInplace(in NativeLayout targetLayout)
+        {
+            int targetNDim = targetLayout._ndim;
+            if (_ndim <= 0 || targetNDim <= 0)
+            {
+                throw new Exception("Cannot broadcast (to) empty tensor shape");
+            }
+
+            if (IsScalar())
+            {
+                _ndim = targetNDim;
+                for (int i = 0; i < targetNDim; i++)
+                {
+                    Shape[i] = targetLayout.Shape[i];
+                    Stride[i] = targetLayout.Shape[i] == 1 ? 1 : 0;
+                }
+                return;
+            }
+
+            if (targetNDim < _ndim)
+            {
+                throw new Exception($"Dimension after broadcast is less than that before braodcast. ");
+            }
+
+            for (int i = 0; i < targetNDim; i++)
+            {
+                int targetIdx = targetNDim - i - 1;
+                int cur_shape = i < _ndim ? Shape[_ndim - i - 1] : 1, cur_stride = i < _ndim ? Stride[_ndim - i - 1] : 0;
+                if (targetLayout.Shape[targetIdx] != cur_shape)
+                {
+                    if (cur_shape != 1 && cur_stride != 0)
+                    {
+                        throw new Exception($"Broadcast on dim {_ndim - i - 1} with shape not equal to 0 or 1.");
+                    }
+                    Shape[targetIdx] = targetLayout.Shape[targetIdx];
+                    Stride[targetIdx] = 0;
+                }
+                else
+                {
+                    Shape[targetIdx] = cur_shape;
+                    Stride[targetIdx] = cur_stride;
+                }
+            }
+            _ndim = targetNDim;
+        }
+
+        public NativeLayout Broadcast(in NativeLayout targetLayout)
+        {
+            NativeLayout res = new NativeLayout(this);
+            res.BroadcastInplace(targetLayout);
+            return res;
+        }
+
+        public NativeLayout CollapseContiguous()
+        {
+            if (_ndim == 0)
+            {
+                throw new InvalidShapeException($"The ndim of the tensor that try to collapse contiguously is 0.");
+            }
+            NativeLayout res = new NativeLayout(this);
+
+            // remove all dims with shape 1
+            for (int i = 0; i <= res._ndim - 1 && res._ndim >= 2; i++)
+            {
+                if (res.Shape[i] == 0)
+                {
+                    // empty tensor
+                    res._ndim = 1;
+                    res.Shape[0] = 0;
+                    res.Stride[0] = 1;
+                    return res;
+                }
+                if (res.Shape[i] == 1) res.RemoveAxisInplace(i);
+            }
+
+            if (res._ndim == 1)
+            {
+                if (res.Shape[0] <= 1)
+                {
+                    res.Stride[0] = 1;
+                }
+                return res;
+            }
+
+            if (res._ndim <= 0 || res.Shape[res._ndim - 1] <= 0)
+            {
+                throw new InvalidShapeException("CollapseContiguous");
+            }
+            for (int i = res._ndim - 2; i >= 0; i--)
+            {
+                if (res.Shape[i] <= 0)
+                {
+                    throw new InvalidShapeException("CollapseContiguous");
+                }
+                if (res.Stride[i] == res.Stride[i + 1] * res.Shape[i + 1])
+                {
+                    res.Shape[i] *= res.Shape[i + 1];
+                    res.Stride[i] = res.Stride[i + 1];
+                    res.RemoveAxisInplace(i + 1);
+                }
+            }
+            return res;
+        }
+
+        public void RemoveDanglingAxesInplace()
+        {
+            for (int i = _ndim - 1; i >= 0; i--)
+            {
+                if (Shape[i] == 1 && _ndim > 1)
+                {
+                    RemoveAxisInplace(i);
+                }
+            }
+        }
+
         public override string ToString()
         {
             string res = "(";
@@ -152,6 +296,12 @@ public override string ToString()
             res += ")";
             return res;
         }
+
+        public static NativeLayout ShapeLike(in NativeLayout layout)
+        {
+            return new NativeLayout(layout._shape.AsSpan(0, layout._ndim));
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static Enumerator GetIndexEnumerator(in NativeLayout layout) => new Enumerator(layout);
 
@@ -164,6 +314,13 @@ public ref struct Enumerator
             private int _ndim;
             private int _currentIndex;
 
+            public int CurrentIndex { get => _currentIndex; }
+
+            public void Clear()
+            {
+                _currentIndex = _index1 = _index2 = _index3;
+            }
+
             [MethodImpl (MethodImplOptions.AggressiveInlining)]
             public Enumerator(in NativeLayout layout)
             {
@@ -217,6 +374,7 @@ public int MoveNext()
             {
                 return _ndim switch
                 {
+                    0 => 0,
                     1 => MoveWithNDim1(),
                     2 => MoveWithNDim2(),
                     3 => MoveWithNDim3(),
@@ -240,6 +398,7 @@ private int MoveWithNDim2()
                 if(_index1 >= _shape1)
                 {
                     _currentIndex += _stride0 - _product1;
+                    _index1 = 0;
                 }
                 return res;
             }
@@ -253,10 +412,12 @@ private int MoveWithNDim3()
                 {
                     _index1++;
                     _currentIndex += _stride1 - _product2;
+                    _index2 = 0;
                 }
                 if (_index1 >= _shape1)
                 {
                     _currentIndex += _stride0 - _product1;
+                    _index1 = 0;
                 }
                 return res;
             }
@@ -270,15 +431,18 @@ private int MoveWithNDim4()
                 {
                     _index2++;
                     _currentIndex += _stride2 - _product3;
+                    _index3 = 0;
                 }
                 if (_index2 >= _shape2)
                 {
                     _index1++;
                     _currentIndex += _stride1 - _product2;
+                    _index2 = 0;
                 }
                 if (_index1 >= _shape1)
                 {
                     _currentIndex += _stride0 - _product1;
+                    _index1 = 0;
                 }
                 return res;
             }
diff --git a/NN/Data/NativeArray.cs b/NN/Data/NativeArray.cs
index ed384bf..aab4215 100644
--- a/NN/Data/NativeArray.cs
+++ b/NN/Data/NativeArray.cs
@@ -1,9 +1,12 @@
 ﻿using NN.Native.Abstraction.Data;
 using NN.Native.Basic;
+using NN.Native.Extensions;
+using NN.Native.Exceptions;
 using System;
 using System.Buffers;
 using System.Collections.Generic;
 using System.Linq;
+using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Text;
@@ -18,10 +21,23 @@ public struct NativeArray<T> where T: unmanaged
         
         public Span<T> Span { get => _storage._memory.Span; }
 
-        public NativeArray(NativeLayout layout, INativeMemoryManager memoryManager)
+        public NativeArray(NativeLayout layout, INativeMemoryManager? memoryManager)
         {
             _layout = layout;
-            _storage = new NativeStorage<T>(memoryManager.AllocateMemory<T>(_layout.TotalElemCount()), 0);
+            if(memoryManager is not null)
+            {
+                _storage = new NativeStorage<T>(memoryManager.AllocateMemory<T>(_layout.TotalElemCount()), 0);
+            }
+            else
+            {
+                _storage = new NativeStorage<T>(INativeMemoryManager.Default.AllocateMemory<T>(_layout.TotalElemCount()), 0);
+            }
+        }
+
+        public NativeArray(NativeLayout layout, NativeStorage<T> storage)
+        {
+            _layout = layout;
+            _storage = storage;
         }
 
         public NativeArray(NativeLayout layout, Memory<T> memory)
@@ -37,6 +53,108 @@ public unsafe MemoryHandle Pin()
 
         public readonly bool IsMutable() => _storage._mutable;
         public readonly bool IsMemoryOwner() => _storage._owned;
+
+        public T this[params int[] indices]
+        {
+            get
+            {
+                if(indices.Length != _layout._ndim)
+                {
+                    throw new InvalidShapeException();
+                }
+                return Span[_layout.IndicesToOffset(indices)];
+            }
+            set
+            {
+                if (indices.Length != _layout._ndim)
+                {
+                    throw new InvalidShapeException();
+                }
+                Span[_layout.IndicesToOffset(indices)] = value;
+            }
+        }
+
+        public override string ToString()
+        {
+            var layout = _layout;
+            var indexEnumerator = NativeLayout.GetIndexEnumerator(layout);
+
+            StringBuilder r = new StringBuilder($"NativeArray {layout}:\n");
+            var span = Span;
+            for (int i = 0; i < layout.TotalElemCount(); i++)
+            {
+                int mod = 1;
+                for (int j = layout._ndim - 1; j >= 0; j--)
+                {
+                    mod *= layout._shape[j];
+                    if (i % mod == 0)
+                    {
+                        r.Append("[");
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+                r.Append(" ").Append(span[indexEnumerator.MoveNext()]);
+
+                if ((i + 1) % layout._shape[layout._ndim - 1] != 0) r.Append(",");
+
+                r.Append(" ");
+                mod = 1;
+                int hit_times = 0;
+                for (int j = layout._ndim - 1; j >= 0; j--)
+                {
+                    mod *= layout._shape[j];
+                    if ((i + 1) % mod == 0)
+                    {
+                        r.Append("]");
+                        hit_times++;
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+                if (hit_times > 0 && hit_times < layout._ndim)
+                {
+                    r.Append(",\n");
+                    for (int j = 0; j < layout._ndim - hit_times; j++)
+                    {
+                        r.Append(" ");
+                    }
+                }
+            }
+            // r.Append("\n");
+            return r.ToString();
+        }
+
+        public static explicit operator NativeArray<int>(NativeArray<T> src)
+        {
+            if(typeof(T) != typeof(int))
+            {
+                throw new BadTypeException();
+            }
+            return new NativeArray<int>(new NativeLayout(src._layout), (NativeStorage<int>)(ValueType)src._storage);
+        }
+
+        public static explicit operator NativeArray<float>(NativeArray<T> src)
+        {
+            if (typeof(T) != typeof(float))
+            {
+                throw new BadTypeException();
+            }
+            return new NativeArray<float>(new NativeLayout(src._layout), (NativeStorage<float>)(ValueType)src._storage);
+        }
+
+        public static explicit operator NativeArray<double>(NativeArray<T> src)
+        {
+            if (typeof(T) != typeof(double))
+            {
+                throw new BadTypeException();
+            }
+            return new NativeArray<double>(new NativeLayout(src._layout), (NativeStorage<double>)(ValueType)src._storage);
+        }
     }
 
     public partial class NativeArray
@@ -69,6 +187,15 @@ public static NativeArray<T> Normal<T>(NativeLayout layout, T minValue, T maxVal
                         span[i] = ((float)random.NextDouble()) * ((float)(object)maxValue - (float)(object)minValue) + (float)(object)minValue;
                     }
                 }
+                else if (typeof(T) == typeof(double))
+                {
+                    var span = MemoryMarshal.Cast<T, double>(res.Span);
+                    var random = new System.Random();
+                    for (int i = 0; i < span.Length; i++)
+                    {
+                        span[i] = ((double)random.NextDouble()) * ((double)(object)maxValue - (double)(object)minValue) + (double)(object)minValue;
+                    }
+                }
                 else
                 {
                     throw new NotImplementedException();
diff --git a/NN/Data/NativeStorage.cs b/NN/Data/NativeStorage.cs
index 8dde00b..d38e19e 100644
--- a/NN/Data/NativeStorage.cs
+++ b/NN/Data/NativeStorage.cs
@@ -7,7 +7,7 @@
 
 namespace NN.Native.Data
 {
-    internal struct NativeStorage<T> where T: unmanaged
+    public struct NativeStorage<T> where T: unmanaged
     {
         internal Memory<T> _memory;
         internal int _offset;
diff --git a/NN/Exceptions/BadTypeException.cs b/NN/Exceptions/BadTypeException.cs
new file mode 100644
index 0000000..291192d
--- /dev/null
+++ b/NN/Exceptions/BadTypeException.cs
@@ -0,0 +1,14 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Exceptions
+{
+    public class BadTypeException : Exception
+    {
+        public BadTypeException(string? message) : base(message) { }
+        public BadTypeException() : base() { }
+    }
+}
diff --git a/NN/Exceptions/InvalidShapeException.cs b/NN/Exceptions/InvalidShapeException.cs
new file mode 100644
index 0000000..2f07f30
--- /dev/null
+++ b/NN/Exceptions/InvalidShapeException.cs
@@ -0,0 +1,14 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Exceptions
+{
+    public class InvalidShapeException : Exception
+    {
+        public InvalidShapeException(string? message) : base(message) { }
+        public InvalidShapeException() : base() { }
+    }
+}
diff --git a/NN/Extensions/ArrayEqualExtension.cs b/NN/Extensions/ArrayEqualExtension.cs
new file mode 100644
index 0000000..8a9172c
--- /dev/null
+++ b/NN/Extensions/ArrayEqualExtension.cs
@@ -0,0 +1,50 @@
+﻿using NN.Native.Basic;
+using NN.Native.Data;
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Extensions
+{
+    public static class ArrayEqualExtension
+    {
+        public static bool IsElementsEqualWith(this NativeArray<float> array, NativeArray<float> other)
+        {
+            return AreEqual(array, other, (x, y) => MathF.Abs(x - y) < 0.01f);
+        }
+        public static bool IsElementsEqualWith(this NativeArray<double> array, NativeArray<double> other)
+        {
+            return AreEqual(array, other, (x, y) => Math.Abs(x - y) < 0.0001);
+        }
+        public static bool IsElementsEqualWith(this NativeArray<int> array, NativeArray<int> other)
+        {
+            return AreEqual(array, other, (x, y) => x == y);
+        }
+        private static bool AreEqual<T>(in NativeArray<T> a, in NativeArray<T> b, Func<T, T, bool> equalFunc) where T: unmanaged
+        {
+            if(!a._layout.IsSameShape(b._layout))
+            {
+                return false;
+            }
+            var totalCount = a._layout.TotalElemCount();
+            
+            var enumeratorA = NativeLayout.GetIndexEnumerator(a._layout);
+            var enumeratorB = NativeLayout.GetIndexEnumerator(b._layout);
+            var spanA = a.Span;
+            var spanB = b.Span;
+            for(int i = 0; i < totalCount; i++)
+            {
+                if (!equalFunc(spanA[enumeratorA.MoveNext()], spanB[enumeratorB.MoveNext()]))
+                {
+                    Debug.WriteLine($"Two arrays differ at offset {i}. {a._layout.GetOffsetDescription(i)}");
+                    return false;
+                }
+            }
+            return true;
+        }
+    }
+}
diff --git a/NN/Extensions/IndexOffsetConvertExtension.cs b/NN/Extensions/IndexOffsetConvertExtension.cs
index d4f5120..75417f7 100644
--- a/NN/Extensions/IndexOffsetConvertExtension.cs
+++ b/NN/Extensions/IndexOffsetConvertExtension.cs
@@ -30,5 +30,31 @@ public unsafe static void OffsetToIndices(this NativeLayout layout, int* indices
                 offset -= idx * layout._stride[i];
             }
         }
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public unsafe static int IndicesToOffset(this NativeLayout layout, int[] indices)
+        {
+            int res = 0;
+            for (int i = 0; i < layout._ndim; i++)
+            {
+                res += indices[i] * layout._stride[i];
+            }
+            return res;
+        }
+        [method: MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public unsafe static void OffsetToIndices(this NativeLayout layout, int[] indices, int offset)
+        {
+            for (int i = 0; i < layout._ndim; i++)
+            {
+                int idx = offset / layout._stride[i];
+                indices[i] = idx;
+                offset -= idx * layout._stride[i];
+            }
+        }
+        public unsafe static string GetOffsetDescription(this NativeLayout layout, int offset)
+        {
+            var indices = new int[layout._ndim];
+            layout.OffsetToIndices(indices, offset);
+            return $"LayoutInfo: {layout}, CurrentIndex: ({string.Join(',', indices)})";
+        }
     }
 }
diff --git a/NN/Extensions/SpanExtension.cs b/NN/Extensions/SpanExtension.cs
new file mode 100644
index 0000000..1e7c69c
--- /dev/null
+++ b/NN/Extensions/SpanExtension.cs
@@ -0,0 +1,35 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Extensions
+{
+    internal static class SpanExtension
+    {
+        public static void Print2D<T>(this ReadOnlySpan<T> span, int cols)
+        {
+            for(int i = 0; i < span.Length / cols; i++)
+            {
+                for(int j = 0; j < cols; j++)
+                {
+                    Console.Write(span[i * cols + j] + ", ");
+                }
+                Console.WriteLine();
+            }
+        }
+        public static unsafe void Print2D<T>(this Span<T> span, int cols)
+        {
+            for (int i = 0; i < span.Length / cols; i++)
+            {
+                for (int j = 0; j < cols; j++)
+                {
+                    Console.Write(span[i * cols + j] + ", ");
+                }
+                Console.WriteLine();
+            }
+        }
+    }
+}
diff --git a/NN/NN.Native.csproj b/NN/NN.Native.csproj
index 47a541a..807ecc8 100644
--- a/NN/NN.Native.csproj
+++ b/NN/NN.Native.csproj
@@ -9,7 +9,19 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="System.Memory" Version="4.5.4" Condition="'$(TargetFramework)'=='netstandard2.0'"/>
+    <PackageReference Include="System.Memory" Version="4.5.4" Condition="'$(TargetFramework)'=='netstandard2.0'" />
+  </ItemGroup>
+  
+  <ItemGroup Condition="'$(TargetFramework)' == 'netcoreapp3.1'">
+    <PackageReference Include="IsExternalInit" Version="1.0.3" PrivateAssets="all" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(TargetFramework)'=='netstandard2.0'">
+    <PackageReference Include="IsExternalInit" Version="1.0.3" PrivateAssets="all" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="Operators\Common\" />
   </ItemGroup>
 
 </Project>
diff --git a/NN/Operators/Common/Params/MatmulParam.cs b/NN/Operators/Common/Params/MatmulParam.cs
deleted file mode 100644
index 8134896..0000000
--- a/NN/Operators/Common/Params/MatmulParam.cs
+++ /dev/null
@@ -1,14 +0,0 @@
-﻿using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace NN.Native.Operators.Common.Params
-{
-    public struct MatmulParam
-    {
-        public bool TransposeA;
-        public bool TransposeB;
-    }
-}
diff --git a/NN/Operators/Common/Params/TypeConvertParam.cs b/NN/Operators/Common/Params/TypeConvertParam.cs
deleted file mode 100644
index 09aef48..0000000
--- a/NN/Operators/Common/Params/TypeConvertParam.cs
+++ /dev/null
@@ -1,15 +0,0 @@
-﻿using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace NN.Native.Operators.Common.Params
-{
-    public struct TypeConvertParam
-    {
-        public bool Transpose;
-        public int DimA;
-        public int DimB;
-    }
-}
diff --git a/NN/Operators/Common/TypeConvertOperatorBase.cs b/NN/Operators/Common/TypeConvertOperatorBase.cs
deleted file mode 100644
index 1195f6b..0000000
--- a/NN/Operators/Common/TypeConvertOperatorBase.cs
+++ /dev/null
@@ -1,19 +0,0 @@
-﻿using NN.Native.Abstraction.Operators;
-using NN.Native.Basic;
-using NN.Native.Operators.Common.Params;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace NN.Native.Operators.Common
-{
-    public abstract class TypeConvertOperatorBase
-    {
-        public static void DeduceLayout(in NativeLayout src, in TypeConvertParam param, out NativeLayout dst)
-        {
-            dst = new(src._shape);
-        }
-    }
-}
diff --git a/NN/Operators/Naive/BinaryElemWiseOperator.cs b/NN/Operators/Naive/BinaryElemWiseOperator.cs
new file mode 100644
index 0000000..c20371a
--- /dev/null
+++ b/NN/Operators/Naive/BinaryElemWiseOperator.cs
@@ -0,0 +1,50 @@
+﻿using NN.Native.Abstraction.Operators;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Abstraction.Common;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Data;
+
+namespace NN.Native.Operators.Naive
+{
+#if NET7_0_OR_GREATER
+    public class BinaryElemWiseOperator<TA, TB> : IBinaryElemWiseOperator<TA, TB> where TA : unmanaged where TB: unmanaged
+    {
+        public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => false; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+    public class BinaryElemWiseOperator<TA, TB>: IBinaryElemWiseOperator<TA, TB> where TA : unmanaged where TB: unmanaged
+    {
+        public bool IsThreadSafe { get => false; }
+        public bool RequireContiguousArray { get => false; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<TB> Exec(in NativeArray<TA> src, Func<TA, int, TB> elemFunc, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<TB>(IBinaryElemWiseOperator<TA, TB>.DeduceLayout(src._layout), memoryManager);
+            ExecInternal(src.Span, res.Span, src._layout, res._layout, elemFunc); 
+            return res;
+        }
+
+        private static void ExecInternal(ReadOnlySpan<TA> src, Span<TB> dst, in NativeLayout layoutSrc, in NativeLayout layoutDst, Func<TA, int, TB> elemFunc)
+        {
+            Debug.Assert(layoutSrc.IsSameShape(layoutDst));
+            int totalElems = layoutDst.TotalElemCount();
+            var enumratorA = NativeLayout.GetIndexEnumerator(layoutSrc);
+            for (int i = 0; i < totalElems; i++)
+            {
+                dst[i] = elemFunc(src[enumratorA.MoveNext()], i);
+            }
+        }
+    }
+}
diff --git a/NN/Operators/Naive/MatmulOperator.cs b/NN/Operators/Naive/MatmulOperator.cs
index 1751b4d..b551a0f 100644
--- a/NN/Operators/Naive/MatmulOperator.cs
+++ b/NN/Operators/Naive/MatmulOperator.cs
@@ -1,6 +1,8 @@
-﻿using NN.Native.Abstraction.DType;
+﻿using NN.Native.Abstraction.Data;
+using NN.Native.Abstraction.DType;
 using NN.Native.Abstraction.Operators;
 using NN.Native.Basic;
+using NN.Native.Data;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -16,19 +18,30 @@ namespace NN.Native.Operators.Naive
     public class MatmulOperator<T>: IMatmulOperator<T> where T : unmanaged, INumberBase<T>
     {
         public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => false; }
         public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
 #else
-    public class MatmulOperator<T, U> where T : unmanaged where U: INativeDTypeHandler<T>, new()
+    public class MatmulOperator<T, U>: IMatmulOperator<T> where T : unmanaged where U : INativeDTypeHandler<T>, new()
     {
         private static U _handler = new();
+        public bool RequireContiguousArray { get => false; }
         public bool IsThreadSafe { get => false; }
         public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
 #endif
 #if NET7_0_OR_GREATER
-        //[MethodImpl(MethodImplOptions.AggressiveOptimization)]
         static
 #endif
-        public unsafe void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        public NativeArray<T> Exec(in NativeArray<T> a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null)
+        {
+            var layoutA = new NativeLayout(a._layout);
+            var layoutB = new NativeLayout(b._layout);
+            var layoutC = IMatmulOperator<T>.DeduceLayout(ref layoutA, ref layoutB);
+            var res = new NativeArray<T>(layoutC, memoryManager);
+            ExecInternal(a.Span, b.Span, res.Span, layoutA, layoutB, layoutC);
+            return res;
+        }
+
+        private static void ExecInternal(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
         {
             // The array should be contiguous here
             int aRows = layoutA._shape[0];
diff --git a/NN/Operators/Naive/ReduceOperator.cs b/NN/Operators/Naive/ReduceOperator.cs
new file mode 100644
index 0000000..93ccbcd
--- /dev/null
+++ b/NN/Operators/Naive/ReduceOperator.cs
@@ -0,0 +1,106 @@
+﻿using NN.Native.Abstraction.Common;
+using NN.Native.Abstraction.Data;
+using NN.Native.Abstraction.Operators;
+using NN.Native.Basic;
+using NN.Native.Data;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Operators.Naive
+{
+    public class ReduceOperator<T, TReduceHandler>: IReduceOperator<T, TReduceHandler> where T : unmanaged where TReduceHandler : IReduceOpHandler<T>, new()
+    {
+#if NET7_0_OR_GREATER
+        public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => false; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+        private static TReduceHandler _reduceHandler = new();
+        public bool IsThreadSafe { get => false; }
+        public bool RequireContiguousArray { get => false; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
+
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(in NativeArray<T> src, in ReduceParam param, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(IReduceOperator<T, TReduceHandler>.DeduceLayout(src._layout, param), memoryManager);
+            var reducedLayout = new NativeLayout();
+            var keepedLayout = new NativeLayout();
+            bool[] reducedDims = new bool[src._layout._ndim];
+
+            Debug.Assert(param.Dims.Length <= src._layout._ndim);
+            for (int i = 0; i < param.Dims.Length; i++)
+            {
+                Debug.Assert(param.Dims[i] < src._layout._ndim);
+                reducedDims[param.Dims[i]] = true;
+            }
+            for(int i = 0; i < reducedDims.Length; i++)
+            {
+                if (reducedDims[i])
+                {
+                    reducedLayout.AddAxisInplace(reducedLayout._ndim, src._layout._shape[i], src._layout._stride[i]);
+                }
+                else
+                {
+                    keepedLayout.AddAxisInplace(keepedLayout._ndim, src._layout._shape[i], src._layout._stride[i]);
+                }
+            }
+
+            int reducedTotalElems = reducedLayout.TotalElemCount();
+            int keepedTotalElems = keepedLayout.TotalElemCount();
+            var enumeratorKeeped = NativeLayout.GetIndexEnumerator(keepedLayout);
+            var enumeratorReduced = NativeLayout.GetIndexEnumerator(reducedLayout);
+            var srcSpan = src.Span;
+            var dstSpan = res.Span;
+            T size = ConvertIntToT(reducedTotalElems);
+            if (reducedTotalElems == 0) return res;
+            for(int i = 0, offset = 0; i < keepedTotalElems; i++, offset++)
+            {
+                var keepedPartIdx = enumeratorKeeped.MoveNext();
+                enumeratorReduced.Clear();
+                var value = srcSpan[keepedPartIdx + enumeratorReduced.MoveNext()];
+                for(int j = 1; j < reducedTotalElems; j++)
+                {
+#if NET7_0_OR_GREATER
+                    value = TReduceHandler.Apply(value, TReduceHandler.GetValue(srcSpan[keepedPartIdx + enumeratorReduced.MoveNext()]));
+#else
+                    value = _reduceHandler.Apply(value, _reduceHandler.GetValue(srcSpan[keepedPartIdx + enumeratorReduced.MoveNext()]));
+#endif
+                }
+#if NET7_0_OR_GREATER
+                dstSpan[offset] = TReduceHandler.SetValue(value, size);
+#else
+                dstSpan[offset] = _reduceHandler.SetValue(value, size);
+#endif
+            }
+            return res;
+        }
+
+        private static T ConvertIntToT(int size)
+        {
+            if(typeof(T) == typeof(int))
+            {
+                return (T)(object)size;
+            }
+            else if (typeof(T) == typeof(float))
+            {
+                return (T)(object)(float)size;
+            }
+            else if (typeof(T) == typeof(double))
+            {
+                return (T)(object)(double)size;
+            }
+            else
+            {
+                throw new NotImplementedException();
+            }
+        }
+    }
+}
diff --git a/NN/Operators/Naive/TernaryElemWiseOperator.cs b/NN/Operators/Naive/TernaryElemWiseOperator.cs
new file mode 100644
index 0000000..977fc77
--- /dev/null
+++ b/NN/Operators/Naive/TernaryElemWiseOperator.cs
@@ -0,0 +1,94 @@
+﻿using NN.Native.Abstraction.Operators;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Abstraction.Common;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Data;
+
+namespace NN.Native.Operators.Naive
+{
+#if NET7_0_OR_GREATER
+    public class TernaryElemWiseOperator<TA, TB, TC> : ITernaryElemWiseOperator<TA, TB, TC> where TA : unmanaged where TB : unmanaged where TC : unmanaged
+    {
+        public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => false; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+    public class TernaryElemWiseOperator<TA, TB, TC>: ITernaryElemWiseOperator<TA, TB, TC> where TA : unmanaged where TB: unmanaged where TC : unmanaged
+    {
+        public bool IsThreadSafe { get => false; }
+        public bool RequireContiguousArray { get => false; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<TC> Exec(in NativeArray<TA> a, in NativeArray<TB> b, Func<TA, TB, int, TC> elemFunc, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<TC>(ITernaryElemWiseOperator<TA, TB, TC>.DeduceLayout(a._layout, b._layout), memoryManager);
+            ExecInternal(a.Span, b.Span, res.Span, a._layout, b._layout, res._layout, elemFunc);
+            return res;
+        }
+
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<TC> Exec(in NativeArray<TA> a, TB b, Func<TA, TB, int, TC> elemFunc, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<TC>(ITernaryElemWiseOperator<TA, TB, TC>.DeduceLayout(a._layout), memoryManager);
+            ExecInternal(a.Span, b, res.Span, a._layout, res._layout, elemFunc);
+            return res;
+        }
+
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<TC> Exec(TA a, in NativeArray<TB> b, Func<TA, TB, int, TC> elemFunc, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<TC>(ITernaryElemWiseOperator<TA, TB, TC>.DeduceLayout(b._layout), memoryManager);
+            ExecInternal(a, b.Span, res.Span, b._layout, res._layout, elemFunc);
+            return res;
+        }
+
+        static private unsafe void ExecInternal(ReadOnlySpan<TA> a, ReadOnlySpan<TB> b, Span<TC> dst, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC, Func<TA, TB, int, TC> elemFunc)
+        {
+            Debug.Assert(layoutA.IsSameShape(layoutB));
+            Debug.Assert(layoutA.IsSameShape(layoutC));
+            int totalElems = layoutC.TotalElemCount();
+            var enumeratorA = NativeLayout.GetIndexEnumerator(layoutA);
+            var enumeratorB = NativeLayout.GetIndexEnumerator(layoutB);
+            for (int i = 0; i < totalElems; i++)
+            {
+                dst[i] = elemFunc(a[enumeratorA.MoveNext()], b[enumeratorB.MoveNext()], i);
+            }
+        }
+
+        static private unsafe void ExecInternal(ReadOnlySpan<TA> a, TB b, Span<TC> dst, in NativeLayout layoutA, in NativeLayout layoutC, Func<TA, TB, int, TC> elemFunc)
+        {
+            Debug.Assert(layoutA.IsSameShape(layoutC));
+            int totalElems = layoutC.TotalElemCount();
+            var enumeratorA = NativeLayout.GetIndexEnumerator(layoutA);
+            for (int i = 0; i < totalElems; i++)
+            {
+                dst[i] = elemFunc(a[enumeratorA.MoveNext()], b, i);
+            }
+        }
+
+        static private unsafe void ExecInternal(TA a, ReadOnlySpan<TB> b, Span<TC> dst, in NativeLayout layoutB, in NativeLayout layoutC, Func<TA, TB, int, TC> elemFunc)
+        {
+            Debug.Assert(layoutB.IsSameShape(layoutC));
+            int totalElems = layoutC.TotalElemCount();
+            var enumeratorB = NativeLayout.GetIndexEnumerator(layoutB);
+            for (int i = 0; i < totalElems; i++)
+            {
+                dst[i] = elemFunc(a, b[enumeratorB.MoveNext()], i);
+            }
+        }
+    }
+}
diff --git a/NN/Operators/Naive/TransposeOperator.cs b/NN/Operators/Naive/TransposeOperator.cs
new file mode 100644
index 0000000..4ed56aa
--- /dev/null
+++ b/NN/Operators/Naive/TransposeOperator.cs
@@ -0,0 +1,57 @@
+﻿using NN.Native.Abstraction.Operators;
+using NN.Native.Basic;
+using NN.Native.Data;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Reflection.Metadata;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Extensions;
+using System.Diagnostics;
+using NN.Native.Abstraction.DType;
+using NN.Native.Abstraction.Common;
+using NN.Native.Abstraction.Data;
+
+namespace NN.Native.Operators.Naive
+{
+#if NET7_0_OR_GREATER
+    public class TransposeOperator<T> : ITransposeOperator<T> where T : unmanaged, INumberBase<T>
+    {
+        public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => false; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+    public class TransposeOperator<T> : ITransposeOperator<T> where T : unmanaged
+    {
+        public bool IsThreadSafe { get => false; }
+        public bool RequireContiguousArray { get => false; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(in NativeArray<T> src, in TransposeParam param, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(ITransposeOperator<T>.DeduceLayout(src._layout, param), memoryManager);
+            ExecInternal(src.Span, res.Span, src._layout, res._layout, param);
+            return res;
+        }
+
+        public static unsafe void ExecInternal(ReadOnlySpan<T> src, Span<T> dst, in NativeLayout layoutSrc, in NativeLayout layoutDst, in TransposeParam param)
+        {
+            Debug.Assert(layoutSrc._ndim == layoutDst._ndim);
+            var indices = stackalloc int[layoutSrc._ndim];
+            int totalElems = layoutSrc.TotalElemCount();
+            for (int offset = 0; offset < totalElems; offset++)
+            {
+                layoutSrc.OffsetToIndices(indices, offset);
+                indices[param.DimA] ^= indices[param.DimB];
+                indices[param.DimB] ^= indices[param.DimA];
+                indices[param.DimA] ^= indices[param.DimB];
+                dst[layoutDst.IndicesToOffset(indices)] = src[offset];
+            }
+        }
+    }
+}
diff --git a/NN/Operators/Naive/TypeConvertOperator.cs b/NN/Operators/Naive/TypeConvertOperator.cs
index 39f5d79..aab0a54 100644
--- a/NN/Operators/Naive/TypeConvertOperator.cs
+++ b/NN/Operators/Naive/TypeConvertOperator.cs
@@ -1,23 +1,35 @@
-﻿using NN.Native.Abstraction.DType;
+﻿using NN.Native.Abstraction.Common;
+using NN.Native.Abstraction.Data;
+using NN.Native.Abstraction.DType;
 using NN.Native.Abstraction.Operators;
 using NN.Native.Basic;
+using NN.Native.Data;
 using NN.Native.Extensions;
-using NN.Native.Operators.Common;
-using NN.Native.Operators.Common.Params;
 
 namespace NN.Native.Operators.Naive
 {
-    public class TypeConvertOperator<TA, TB, THandler>: TypeConvertOperatorBase where TA: unmanaged where TB: unmanaged where THandler: INativeConvertible<TA, TB>, new()
+    public class TypeConvertOperator<TA, TB, THandler>: ITypeConvertOperator<TA, TB> where TA: unmanaged where TB: unmanaged where THandler: INativeConvertible<TA, TB>, new()
     {
 #if NET7_0_OR_GREATER
         public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => false; }
         public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
 #else
         private static THandler _handler = new();
         public bool IsThreadSafe { get => false; }
+        public bool RequireContiguousArray { get => false; }
         public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
 #endif
-        public static unsafe void Exec(ReadOnlySpan<TA> a, Span<TB> b, in NativeLayout layoutA, in NativeLayout layoutB, in TypeConvertParam param)
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<TB> Exec(in NativeArray<TA> a, in TypeConvertParam param, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<TB>(ITypeConvertOperator<TA, TB>.DeduceLayout(a._layout, param), memoryManager);
+            ExecInternal(a.Span, res.Span, a._layout, res._layout, param);
+            return res;
+        }
+        public static unsafe void ExecInternal(ReadOnlySpan<TA> a, Span<TB> b, in NativeLayout layoutA, in NativeLayout layoutB, in TypeConvertParam param)
         {
             int totalElems = layoutA.TotalElemCount();
             if (!param.Transpose)
diff --git a/NN/Operators/Naive/UnaryElemWiseOperator.cs b/NN/Operators/Naive/UnaryElemWiseOperator.cs
new file mode 100644
index 0000000..2bbacb4
--- /dev/null
+++ b/NN/Operators/Naive/UnaryElemWiseOperator.cs
@@ -0,0 +1,49 @@
+﻿using NN.Native.Abstraction.Operators;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Abstraction.Common;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Data;
+
+namespace NN.Native.Operators.Naive
+{
+#if NET7_0_OR_GREATER
+    public class UnaryElemWiseOperator<T> : IUnaryElemWiseOperator<T> where T : unmanaged
+    {
+        public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => false; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+    public class UnaryElemWiseOperator<T>: IUnaryElemWiseOperator<T> where T : unmanaged
+    {
+        public bool IsThreadSafe { get => false; }
+        public bool RequireContiguousArray { get => false; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public void Exec(ref NativeArray<T> src, Func<T, int, T> elemFunc)
+        {
+            ExecInternal(src.Span, src._layout, elemFunc);
+        }
+
+        static private unsafe void ExecInternal(Span<T> src, in NativeLayout layoutSrc, Func<T, int, T> elemFunc)
+        {
+            Debug.Assert(!layoutSrc.IsEmpty());
+            int totalElems = layoutSrc.TotalElemCount();
+            var enumratorA = NativeLayout.GetIndexEnumerator(layoutSrc);
+            for (int i = 0; i < totalElems; i++)
+            {
+                var idx = enumratorA.MoveNext();
+                src[idx] = elemFunc(src[idx], i);
+            }
+        }
+    }
+}
diff --git a/NN/Operators/X86/Arithmetic/AddOperator.cs b/NN/Operators/X86/Arithmetic/AddOperator.cs
new file mode 100644
index 0000000..9bd762e
--- /dev/null
+++ b/NN/Operators/X86/Arithmetic/AddOperator.cs
@@ -0,0 +1,114 @@
+﻿using NN.Native.Abstraction.Operators;
+using NN.Native.Abstraction.Operators.Arithmetic;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Abstraction.DType;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Numerics;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Data;
+using System.Buffers;
+
+namespace NN.Native.Operators.X86.Arithmetic
+{
+#if NET7_0_OR_GREATER
+    public class AddOperator<T> : IAddOperator<T> where T : unmanaged, IAdditionOperators<T, T, T>, INumberBase<T>
+    {
+        public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => true; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+    public class AddOperator<T, THandler>: IAddOperator<T> where T : unmanaged where THandler: INativeDTypeHandler<T>, new()
+    {
+        private static THandler _handler = new();
+        public bool IsThreadSafe { get => false; }
+        public bool RequireContiguousArray { get => true; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
+
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(in NativeArray<T> a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(IAddOperator<T>.DeduceLayout(a._layout, b._layout), memoryManager);
+            ExecInternal(a.Span, b.Span, res.Span, a._layout, b._layout, res._layout);
+            return res;
+        }
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(in NativeArray<T> a, T b, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(IAddOperator<T>.DeduceLayout(a._layout), memoryManager);
+            ExecInternal(a.Span, b, res.Span, a._layout, res._layout);
+            return res;
+        }
+        private static unsafe void ExecInternal(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        {
+            Debug.Assert(layoutA.IsSameShape(layoutB));
+            Debug.Assert(layoutA.IsSameShape(layoutC));
+            int length = Vector<T>.Count;
+            int totalElems = layoutC.TotalElemCount();
+            int groupCount = totalElems / length;
+            MicroKernel8xN(a, b, c, groupCount, length);
+            for(int i = groupCount * length; i < totalElems; i++)
+            {
+#if NET7_0_OR_GREATER
+                c[i] = a[i] + b[i];
+#else
+                c[i] = _handler.Add(a[i], b[i]);
+#endif
+            }
+        }
+
+        private static unsafe void ExecInternal(ReadOnlySpan<T> a, T b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutC)
+        {
+            Debug.Assert(layoutA.IsSameShape(layoutC));
+            int length = Vector<T>.Count;
+            int totalElems = layoutC.TotalElemCount();
+            int groupCount = totalElems / length;
+            MicroKernel8xN(a, b, c, groupCount, length);
+            for (int i = groupCount * length; i < totalElems; i++)
+            {
+#if NET7_0_OR_GREATER
+                c[i] = a[i] + b;
+#else
+                c[i] = _handler.Add(a[i], b);
+#endif
+            }
+        }
+
+        private static unsafe void MicroKernel8xN(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, int loops, int length)
+        {
+            Vector<T> a_v, b_v, c_v;
+            int offset = 0;
+            for (int i = 0; i < loops; i++, offset += length)
+            {
+                a_v = new Vector<T>(a.Slice(offset));
+                b_v = new Vector<T>(b.Slice(offset));
+                c_v = Vector.Add(a_v, b_v);
+                c_v.CopyTo(c.Slice(offset));
+            }
+        }
+
+        private static unsafe void MicroKernel8xN(ReadOnlySpan<T> a, T b, Span<T> c, int loops, int length)
+        {
+            Vector<T> a_v, b_v, c_v;
+            b_v = new Vector<T>(b);
+            int offset = 0;
+            for (int i = 0; i < loops; i++, offset += length)
+            {
+                a_v = new Vector<T>(a.Slice(offset));
+                c_v = Vector.Add(a_v, b_v);
+                c_v.CopyTo(c.Slice(offset));
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/NN/Operators/X86/Arithmetic/DivideOperator.cs b/NN/Operators/X86/Arithmetic/DivideOperator.cs
new file mode 100644
index 0000000..abb0df5
--- /dev/null
+++ b/NN/Operators/X86/Arithmetic/DivideOperator.cs
@@ -0,0 +1,152 @@
+﻿using NN.Native.Abstraction.Operators;
+using NN.Native.Abstraction.Operators.Arithmetic;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Abstraction.DType;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Numerics;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Data;
+
+namespace NN.Native.Operators.X86.Arithmetic
+{
+#if NET7_0_OR_GREATER
+    public class DivideOperator<T> : IDivideOperator<T> where T : unmanaged, IAdditionOperators<T, T, T>, INumberBase<T>
+    {
+        public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => true; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+    public class DivideOperator<T, THandler>: IDivideOperator<T> where T : unmanaged where THandler: INativeDTypeHandler<T>, new()
+    {
+        private static THandler _handler = new();
+        public bool IsThreadSafe { get => false; }
+        public bool RequireContiguousArray { get => true; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
+
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(in NativeArray<T> a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(IDivideOperator<T>.DeduceLayout(a._layout, b._layout), memoryManager);
+            ExecInternal(a.Span, b.Span, res.Span, a._layout, b._layout, res._layout);
+            return res;
+        }
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(in NativeArray<T> a, T b, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(IDivideOperator<T>.DeduceLayout(a._layout), memoryManager);
+            ExecInternal(a.Span, b, res.Span, a._layout, res._layout);
+            return res;
+        }
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(T a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(IDivideOperator<T>.DeduceLayout(b._layout), memoryManager);
+            ExecInternal(a, b.Span, res.Span, b._layout, res._layout);
+            return res;
+        }
+        private static unsafe void ExecInternal(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        {
+            Debug.Assert(layoutA.IsSameShape(layoutB));
+            Debug.Assert(layoutA.IsSameShape(layoutC));
+            int length = Vector<T>.Count;
+            int totalElems = layoutC.TotalElemCount();
+            int groupCount = totalElems / length;
+            MicroKernel8xN(a, b, c, groupCount, length);
+            for (int i = groupCount * length; i < totalElems; i++)
+            {
+#if NET7_0_OR_GREATER
+                c[i] = a[i] / b[i];
+#else
+                c[i] = _handler.Divide(a[i], b[i]);
+#endif
+            }
+        }
+
+        private static unsafe void ExecInternal(ReadOnlySpan<T> a, T b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutC)
+        {
+            Debug.Assert(layoutA.IsSameShape(layoutC));
+            int length = Vector<T>.Count;
+            int totalElems = layoutC.TotalElemCount();
+            int groupCount = totalElems / length;
+            MicroKernel8xN(a, b, c, groupCount, length);
+            for (int i = groupCount * length; i < totalElems; i++)
+            {
+#if NET7_0_OR_GREATER
+                c[i] = a[i] / b;
+#else
+                c[i] = _handler.Divide(a[i], b);
+#endif
+            }
+        }
+
+        private static unsafe void ExecInternal(T a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutC)
+        {
+            Debug.Assert(layoutA.IsSameShape(layoutC));
+            int length = Vector<T>.Count;
+            int totalElems = layoutC.TotalElemCount();
+            int groupCount = totalElems / length;
+            MicroKernel8xN(a, b, c, groupCount, length);
+            for (int i = groupCount * length; i < totalElems; i++)
+            {
+#if NET7_0_OR_GREATER
+                c[i] = a / b[i];
+#else
+                c[i] = _handler.Divide(a, b[i]);
+#endif
+            }
+        }
+
+        private static unsafe void MicroKernel8xN(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, int loops, int length)
+        {
+            Vector<T> a_v, b_v, c_v;
+            int offset = 0;
+            for (int i = 0; i < loops; i++, offset += length)
+            {
+                a_v = new Vector<T>(a.Slice(offset));
+                b_v = new Vector<T>(b.Slice(offset));
+                c_v = Vector.Divide(a_v, b_v);
+                c_v.CopyTo(c.Slice(offset));
+            }
+        }
+
+        private static unsafe void MicroKernel8xN(ReadOnlySpan<T> a, T b, Span<T> c, int loops, int length)
+        {
+            Vector<T> a_v, b_v, c_v;
+            b_v = new Vector<T>(b);
+            int offset = 0;
+            for (int i = 0; i < loops; i++, offset += length)
+            {
+                a_v = new Vector<T>(a.Slice(offset));
+                c_v = Vector.Divide(a_v, b_v);
+                c_v.CopyTo(c.Slice(offset));
+            }
+        }
+
+        private static unsafe void MicroKernel8xN(T a, ReadOnlySpan<T> b, Span<T> c, int loops, int length)
+        {
+            Vector<T> a_v, b_v, c_v;
+            a_v = new Vector<T>(a);
+            int offset = 0;
+            for (int i = 0; i < loops; i++, offset += length)
+            {
+                b_v = new Vector<T>(b.Slice(offset));
+                c_v = Vector.Divide(a_v, b_v);
+                c_v.CopyTo(c.Slice(offset));
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/NN/Operators/X86/Arithmetic/MultiplyOperator.cs b/NN/Operators/X86/Arithmetic/MultiplyOperator.cs
new file mode 100644
index 0000000..75cedfc
--- /dev/null
+++ b/NN/Operators/X86/Arithmetic/MultiplyOperator.cs
@@ -0,0 +1,113 @@
+﻿using NN.Native.Abstraction.Operators;
+using NN.Native.Abstraction.Operators.Arithmetic;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Abstraction.DType;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Numerics;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Data;
+
+namespace NN.Native.Operators.X86.Arithmetic
+{
+#if NET7_0_OR_GREATER
+    public class MultiplyOperator<T> : IMultiplyOperator<T> where T : unmanaged, IAdditionOperators<T, T, T>, INumberBase<T>
+    {
+        public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => true; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+    public class MultiplyOperator<T, THandler>: IMultiplyOperator<T> where T : unmanaged where THandler: INativeDTypeHandler<T>, new()
+    {
+        private static THandler _handler = new();
+        public bool IsThreadSafe { get => false; }
+        public bool RequireContiguousArray { get => true; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
+
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(in NativeArray<T> a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(IMultiplyOperator<T>.DeduceLayout(a._layout, b._layout), memoryManager);
+            ExecInternal(a.Span, b.Span, res.Span, a._layout, b._layout, res._layout);
+            return res;
+        }
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(in NativeArray<T> a, T b, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(IMultiplyOperator<T>.DeduceLayout(a._layout), memoryManager);
+            ExecInternal(a.Span, b, res.Span, a._layout, res._layout);
+            return res;
+        }
+        private static unsafe void ExecInternal(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        {
+            Debug.Assert(layoutA.IsSameShape(layoutB));
+            Debug.Assert(layoutA.IsSameShape(layoutC));
+            int length = Vector<T>.Count;
+            int totalElems = layoutC.TotalElemCount();
+            int groupCount = totalElems / length;
+            MicroKernel8xN(a, b, c, groupCount, length);
+            for (int i = groupCount * length; i < totalElems; i++)
+            {
+#if NET7_0_OR_GREATER
+                c[i] = a[i] * b[i];
+#else
+                c[i] = _handler.Multiply(a[i], b[i]);
+#endif
+            }
+        }
+
+        private static unsafe void ExecInternal(ReadOnlySpan<T> a, T b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutC)
+        {
+            Debug.Assert(layoutA.IsSameShape(layoutC));
+            int length = Vector<T>.Count;
+            int totalElems = layoutC.TotalElemCount();
+            int groupCount = totalElems / length;
+            MicroKernel8xN(a, b, c, groupCount, length);
+            for (int i = groupCount * length; i < totalElems; i++)
+            {
+#if NET7_0_OR_GREATER
+                c[i] = a[i] * b;
+#else
+                c[i] = _handler.Multiply(a[i], b);
+#endif
+            }
+        }
+
+        private static unsafe void MicroKernel8xN(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, int loops, int length)
+        {
+            Vector<T> a_v, b_v, c_v;
+            int offset = 0;
+            for (int i = 0; i < loops; i++, offset += length)
+            {
+                a_v = new Vector<T>(a.Slice(offset));
+                b_v = new Vector<T>(b.Slice(offset));
+                c_v = Vector.Multiply(a_v, b_v);
+                c_v.CopyTo(c.Slice(offset));
+            }
+        }
+
+        private static unsafe void MicroKernel8xN(ReadOnlySpan<T> a, T b, Span<T> c, int loops, int length)
+        {
+            Vector<T> a_v, b_v, c_v;
+            b_v = new Vector<T>(b);
+            int offset = 0;
+            for (int i = 0; i < loops; i++, offset += length)
+            {
+                a_v = new Vector<T>(a.Slice(offset));
+                c_v = Vector.Multiply(a_v, b_v);
+                c_v.CopyTo(c.Slice(offset));
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/NN/Operators/X86/Arithmetic/NegativeOperator.cs b/NN/Operators/X86/Arithmetic/NegativeOperator.cs
new file mode 100644
index 0000000..a95aa66
--- /dev/null
+++ b/NN/Operators/X86/Arithmetic/NegativeOperator.cs
@@ -0,0 +1,76 @@
+﻿using NN.Native.Abstraction.Operators;
+using NN.Native.Abstraction.Operators.Arithmetic;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Abstraction.DType;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Numerics;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Data;
+using System.Buffers;
+
+namespace NN.Native.Operators.X86.Arithmetic
+{
+#if NET7_0_OR_GREATER
+    public class NegativeOperator<T>: INegativeOperator<T> where T : unmanaged, IAdditionOperators<T, T, T>, INumberBase<T>
+    {
+        public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => true; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+    public class NegativeOperator<T, THandler>: INegativeOperator<T> where T : unmanaged where THandler: INativeDTypeHandler<T>, new()
+    {
+        private static THandler _handler = new();
+        public bool IsThreadSafe { get => false; }
+        public bool RequireContiguousArray { get => true; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
+
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(in NativeArray<T> src, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(IAddOperator<T>.DeduceLayout(src._layout), memoryManager);
+            ExecInternal(src.Span, res.Span, src._layout, res._layout);
+            return res;
+        }
+
+        private static unsafe void ExecInternal(ReadOnlySpan<T> src, Span<T> dst, in NativeLayout layoutSrc, in NativeLayout layoutDst)
+        {
+            Debug.Assert(layoutSrc.IsSameShape(layoutDst));
+            Debug.Assert(layoutSrc.IsSameShape(layoutDst));
+            int length = Vector<T>.Count;
+            int totalElems = layoutDst.TotalElemCount();
+            int groupCount = totalElems / length;
+            MicroKernel8xN(src, dst, groupCount, length);
+            for (int i = groupCount * length; i < totalElems; i++)
+            {
+#if NET7_0_OR_GREATER
+                dst[i] = -src[i];
+#else
+                dst[i] = _handler.GetNegative(src[i]);
+#endif
+            }
+        }
+
+
+        private static unsafe void MicroKernel8xN(ReadOnlySpan<T> src, Span<T> dst, int loops, int length)
+        {
+            Vector<T> src_v, dst_v;
+            int offset = 0;
+            for (int i = 0; i < loops; i++, offset += length)
+            {
+                src_v = new Vector<T>(src.Slice(offset));
+                dst_v = -src_v;
+                dst_v.CopyTo(dst.Slice(offset));
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/NN/Operators/X86/Arithmetic/SubtractOperator.cs b/NN/Operators/X86/Arithmetic/SubtractOperator.cs
new file mode 100644
index 0000000..db4889e
--- /dev/null
+++ b/NN/Operators/X86/Arithmetic/SubtractOperator.cs
@@ -0,0 +1,152 @@
+﻿using NN.Native.Abstraction.Operators;
+using NN.Native.Abstraction.Operators.Arithmetic;
+using NN.Native.Basic;
+using NN.Native.Data;
+using NN.Native.Abstraction.DType;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Numerics;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
+using System.Text;
+using System.Threading.Tasks;
+using NN.Native.Abstraction.Data;
+
+namespace NN.Native.Operators.X86.Arithmetic
+{
+#if NET7_0_OR_GREATER
+    public class SubtractOperator<T> : ISubtractOperator<T> where T : unmanaged, IAdditionOperators<T, T, T>, INumberBase<T>
+    {
+        public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => true; }
+        public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#else
+    public class SubtractOperator<T, THandler>: ISubtractOperator<T> where T : unmanaged where THandler: INativeDTypeHandler<T>, new()
+    {
+        private static THandler _handler = new();
+        public bool IsThreadSafe { get => false; }
+        public bool RequireContiguousArray { get => true; }
+        public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
+#endif
+
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(in NativeArray<T> a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(ISubtractOperator<T>.DeduceLayout(a._layout, b._layout), memoryManager);
+            ExecInternal(a.Span, b.Span, res.Span, a._layout, b._layout, res._layout);
+            return res;
+        }
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(in NativeArray<T> a, T b, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(ISubtractOperator<T>.DeduceLayout(a._layout), memoryManager);
+            ExecInternal(a.Span, b, res.Span, a._layout, res._layout);
+            return res;
+        }
+#if NET7_0_OR_GREATER
+        static
+#endif
+        public NativeArray<T> Exec(T a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null)
+        {
+            var res = new NativeArray<T>(ISubtractOperator<T>.DeduceLayout(b._layout), memoryManager);
+            ExecInternal(a, b.Span, res.Span, b._layout, res._layout);
+            return res;
+        }
+        private static unsafe void ExecInternal(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        {
+            Debug.Assert(layoutA.IsSameShape(layoutB));
+            Debug.Assert(layoutA.IsSameShape(layoutC));
+            int length = Vector<T>.Count;
+            int totalElems = layoutC.TotalElemCount();
+            int groupCount = totalElems / length;
+            MicroKernel8xN(a, b, c, groupCount, length);
+            for (int i = groupCount * length; i < totalElems; i++)
+            {
+#if NET7_0_OR_GREATER
+                c[i] = a[i] - b[i];
+#else
+                c[i] = _handler.Subtract(a[i], b[i]);
+#endif
+            }
+        }
+
+        private static unsafe void ExecInternal(ReadOnlySpan<T> a, T b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutC)
+        {
+            Debug.Assert(layoutA.IsSameShape(layoutC));
+            int length = Vector<T>.Count;
+            int totalElems = layoutC.TotalElemCount();
+            int groupCount = totalElems / length;
+            MicroKernel8xN(a, b, c, groupCount, length);
+            for (int i = groupCount * length; i < totalElems; i++)
+            {
+#if NET7_0_OR_GREATER
+                c[i] = a[i] - b;
+#else
+                c[i] = _handler.Subtract(a[i], b);
+#endif
+            }
+        }
+
+        private static unsafe void ExecInternal(T a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutC)
+        {
+            Debug.Assert(layoutA.IsSameShape(layoutC));
+            int length = Vector<T>.Count;
+            int totalElems = layoutC.TotalElemCount();
+            int groupCount = totalElems / length;
+            MicroKernel8xN(a, b, c, groupCount, length);
+            for (int i = groupCount * length; i < totalElems; i++)
+            {
+#if NET7_0_OR_GREATER
+                c[i] = a - b[i];
+#else
+                c[i] = _handler.Subtract(a, b[i]);
+#endif
+            }
+        }
+
+        private static unsafe void MicroKernel8xN(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, int loops, int length)
+        {
+            Vector<T> a_v, b_v, c_v;
+            int offset = 0;
+            for (int i = 0; i < loops; i++, offset += length)
+            {
+                a_v = new Vector<T>(a.Slice(offset));
+                b_v = new Vector<T>(b.Slice(offset));
+                c_v = Vector.Subtract(a_v, b_v);
+                c_v.CopyTo(c.Slice(offset));
+            }
+        }
+
+        private static unsafe void MicroKernel8xN(ReadOnlySpan<T> a, T b, Span<T> c, int loops, int length)
+        {
+            Vector<T> a_v, b_v, c_v;
+            b_v = new Vector<T>(b);
+            int offset = 0;
+            for (int i = 0; i < loops; i++, offset += length)
+            {
+                a_v = new Vector<T>(a.Slice(offset));
+                c_v = Vector.Subtract(a_v, b_v);
+                c_v.CopyTo(c.Slice(offset));
+            }
+        }
+
+        private static unsafe void MicroKernel8xN(T a, ReadOnlySpan<T> b, Span<T> c, int loops, int length)
+        {
+            Vector<T> a_v, b_v, c_v;
+            a_v = new Vector<T>(a);
+            int offset = 0;
+            for (int i = 0; i < loops; i++, offset += length)
+            {
+                b_v = new Vector<T>(b.Slice(offset));
+                c_v = Vector.Subtract(a_v, b_v);
+                c_v.CopyTo(c.Slice(offset));
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/NN/Operators/X86/DotOperator.cs b/NN/Operators/X86/DotOperator.cs
new file mode 100644
index 0000000..282f73a
--- /dev/null
+++ b/NN/Operators/X86/DotOperator.cs
@@ -0,0 +1,15 @@
+﻿using NN.Native.Abstraction.Data;
+using NN.Native.Abstraction.DType;
+using NN.Native.Abstraction.Operators;
+using NN.Native.Basic;
+using NN.Native.Data;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NN.Native.Operators.X86
+{
+
+}
diff --git a/NN/Operators/X86/MatmulOperator.cs b/NN/Operators/X86/MatmulOperator.cs
index 8a96502..1ba895d 100644
--- a/NN/Operators/X86/MatmulOperator.cs
+++ b/NN/Operators/X86/MatmulOperator.cs
@@ -1,16 +1,12 @@
 ﻿using NN.Native.Abstraction.Operators;
 using NN.Native.Basic;
 using NN.Native.Abstraction.DType;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Numerics;
-using System.Text;
-using System.Threading.Tasks;
-using System.Runtime.CompilerServices;
-using System.ComponentModel.Design.Serialization;
 using System.Runtime.Intrinsics;
 using System.Runtime.Intrinsics.X86;
+using NN.Native.Data;
+using NN.Native.Abstraction.Data;
+using System.Linq.Expressions;
 
 namespace NN.Native.Operators.X86
 {
@@ -18,109 +14,660 @@ namespace NN.Native.Operators.X86
     public class MatmulOperator<T> : IMatmulOperator<T> where T : unmanaged, INumberBase<T>
     {
         public static bool IsThreadSafe { get => false; }
+        public static bool RequireContiguousArray { get => true; }
         public static OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
 #else
-    public class MatmulOperator<T, U>: IMatmulOperator<T> where T : unmanaged where U: INativeDTypeHandler<T>, new()
+    public class MatmulOperator<T, THandler>: IMatmulOperator<T> where T : unmanaged where THandler: INativeDTypeHandler<T>, new()
     {
-        private static U _handler = new();
+        private static THandler _handler = new();
         public bool IsThreadSafe { get => false; }
+        public bool RequireContiguousArray { get => true; }
         public OperatorHandlerType HandlerType { get => OperatorHandlerType.Naive; }
 #endif
+        private static readonly int _bw = 64;
+        private static readonly int _bh = 64;
+
 #if NET7_0_OR_GREATER
-        //[MethodImpl(MethodImplOptions.AggressiveOptimization)]
         static
 #endif
-        public unsafe void Exec(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
+        public NativeArray<T> Exec(in NativeArray<T> a, in NativeArray<T> b, INativeMemoryManager? memoryManager = null)
+        {
+            var layoutA = new NativeLayout(a._layout);
+            var layoutB = new NativeLayout(b._layout);
+            var res = new NativeArray<T>(IMatmulOperator<T>.DeduceLayout(ref layoutA, ref layoutB), memoryManager);
+            ExecInternal(a.Span, b.Span, res.Span, layoutA, layoutB, res._layout);
+            return res;
+        }
+
+        private unsafe static void ExecInternal(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, in NativeLayout layoutA, in NativeLayout layoutB, in NativeLayout layoutC)
         {
             // The array should be contiguous here
             int aRows = layoutA._shape[0];
             int aCols = layoutA._shape[1];
             int bCols = layoutB._shape[1];
+            int bIdx, cIdx = 0;
 
-            int aIdx = 0, bIdx = 0;
-
-            for (int i = 0; i <= aRows - 8; i += 8)
+            var packedB = new T[_bh * _bw];
+            var packedSpanB = packedB.AsSpan();
+            if(typeof(T) == typeof(float))
             {
-                bIdx = 0;
-                for (int j = 0; j <= bCols - 8; j += 8)
+                fixed (T* ptrA = a, ptrB = b, ptrC = c, ptrPackedB = packedSpanB)
                 {
-                    Kernel32b8x8(a.Slice(aIdx), b.Slice(bIdx), c.Slice(i * bCols + j), aRows, aCols, bCols);
-                    bIdx += 8;
+                    float* fa = (float*)ptrA, fb = (float*)ptrB, fc = (float*)ptrC, fPackedB = (float*)ptrPackedB;
+                    for (int j = 0; j < bCols; j += _bw, cIdx += _bw)
+                    {
+                        int width = (bCols - j) > _bw ? _bw : (bCols - j);
+                        for (int k = 0; k < aCols; k += _bh)
+                        {
+                            int height = (aCols - k) > _bh ? _bh : (aCols - k);
+                            bIdx = j + k * bCols;
+                            Float32MicroKernel.ExecBlock(fa + k, fb + bIdx, fc + cIdx, aRows, height, width, aRows, aCols, bCols, fPackedB);
+                        }
+                    }
                 }
-                aIdx += 8 * aCols;
             }
-            for (int i = aRows / 8 * 8; i < aRows; i++)
+            else if(typeof(T) == typeof(double))
             {
-                for (int j = 0; j < bCols; j++)
+                fixed (T* ptrA = a, ptrB = b, ptrC = c, ptrPackedB = packedSpanB)
                 {
-                    for (int k = 0; k < aCols; k++)
+                    double* da = (double*)ptrA, db = (double*)ptrB, dc = (double*)ptrC, dPackedB = (double*)ptrPackedB;
+                    for (int j = 0; j < bCols; j += _bw, cIdx += _bw)
                     {
-#if NET7_0_OR_GREATER
-                        c[i * bCols + j] += a[i * aCols + k] * b[k * bCols + j];
-#endif
+                        int width = (bCols - j) > _bw ? _bw : (bCols - j);
+                        for (int k = 0; k < aCols; k += _bh) // exchange order 
+                        {
+                            int height = (aCols - k) > _bh ? _bh : (aCols - k);
+                            bIdx = j + k * bCols;
+                            Float64MicroKernel.ExecBlock(da + k, db + bIdx, dc + cIdx, aRows, height, width, aRows, aCols, bCols, dPackedB);
+                        }
                     }
                 }
             }
-            for (int i = 0; i < aRows / 8 * 8; i++)
+            else
             {
-                for (int j = bCols / 8 * 8; j < bCols; j++)
+                fixed (T* ptrA = a, ptrB = b, ptrC = c, ptrPackedB = packedSpanB)
                 {
-                    for (int k = 0; k < aCols; k++)
+                    int length = Vector<T>.Count;
+                    for (int j = 0; j < bCols; j += _bw, cIdx += _bw)
                     {
+                        int width = (bCols - j) > _bw ? _bw : (bCols - j);
+                        for (int k = 0; k < aCols; k += _bh) // exchange order 
+                        {
+                            int height = (aCols - k) > _bh ? _bh : (aCols - k);
+                            bIdx = j + k * bCols;
+                            GenericMicroKernel.ExecBlock(ptrA + k, ptrB + bIdx, ptrC + cIdx, aRows, height, width, aRows, aCols, bCols, ptrPackedB, length);
+                        }
+                    }
+                }
+            }
+        }
+
+        private static class Utils
+        {
+            public unsafe static void PackMatrixBWithPxQ<U>(U* src, U* dst, int p, int q, int bCols) where U: unmanaged
+            {
 #if NET7_0_OR_GREATER
-                        c[i * bCols + j] += a[i * aCols + k] * b[k * bCols + j];
+            Vector256<U> data;
+            for (int k = 0; k < p; k++, dst += q, src += bCols)
+            {
+                data = Vector256.Load(src);
+                data.Store(dst);
+            }
+#else
+                for (int k = 0; k < p; k++, src += bCols)
+                {
+                    for(int j = 0; j < q; j++)
+                    {
+                        *(dst++) = src[j];
+                    }
+                }
 #endif
+            }
+        }
+
+        private static class Float32MicroKernel
+        {
+            // (m, p) * (p, n)
+            public static unsafe void ExecBlock(float* a, float* b, float* c, int m, int p, int n, int aRows, int aCols, int bCols, float* packedB)
+            {
+                int aIdx, bIdx, cIdx;
+
+                // 8x8 blocks
+                for (int i = 0; i <= m - 8; i += 8)
+                {
+                    cIdx = i * bCols;
+                    aIdx = i * aCols;
+                    for (int j = 0; j <= n - 8; j += 8, cIdx += 8)
+                    {
+                        bIdx = j * p;
+                        if (i == 0) Utils.PackMatrixBWithPxQ(b + j, packedB + bIdx, p, 8, bCols);
+                        Kernel8x8(a + aIdx, packedB + bIdx, c + cIdx, p, 8, aRows, aCols, bCols);
+                    }
+                }
+                // The left rows and every 8 columns
+                int iStart = m / 8 * 8;
+                aIdx = iStart * aCols;
+                bool hasPacked = m >= 8 && n >= 8;
+                for (int i = iStart; i < m; i++, aIdx += aCols)
+                {
+                    cIdx = i * bCols;
+                    for (int j = 0; j <= n - 8; j += 8, cIdx += 8)
+                    {
+                        if (!hasPacked && i == iStart) Utils.PackMatrixBWithPxQ(b + j, packedB + j * p, p, 8, bCols);
+                        Kernel1x8(a + aIdx, packedB + j * p, c + cIdx, p, 8, aRows, aCols, bCols);
+                    }
+                }
+
+                // The left columns and every 4 rows
+                int jStart = n / 8 * 8;
+                float c0, c1, c2, c3;
+                for (int i = 0; i <= m - 4; i += 4)
+                {
+                    for (int j = jStart; j < n; j++)
+                    {
+                        int aIdx0 = i * aCols, aIdx1 = aIdx0 + aCols, aIdx2 = aIdx1 + aCols, aIdx3 = aIdx2 + aCols;
+                        cIdx = i * bCols + j;
+                        c0 = c[cIdx];
+                        cIdx += bCols;
+                        c1 = c[cIdx];
+                        cIdx += bCols;
+                        c2 = c[cIdx];
+                        cIdx += bCols;
+                        c3 = c[cIdx];
+
+                        for (int k = 0; k < p; k++)
+                        {
+                            var bValue = b[k * bCols + j];
+                            c0 += a[aIdx0++] * bValue;
+                            c1 += a[aIdx1++] * bValue;
+                            c2 += a[aIdx2++] * bValue;
+                            c3 += a[aIdx3++] * bValue;
+                        }
+                        cIdx = i * bCols + j;
+                        c[cIdx] = c0;
+                        cIdx += bCols;
+                        c[cIdx] = c1;
+                        cIdx += bCols;
+                        c[cIdx] = c2;
+                        cIdx += bCols;
+                        c[cIdx] = c3;
+                    }
+                }
+                // The last 1-3 rows and the left columns
+                for (int i = m / 4 * 4; i < m; i++)
+                {
+                    for (int k = 0; k < p; k++)
+                    {
+                        cIdx = i * bCols + jStart;
+                        bIdx = k * bCols + jStart;
+                        var aValue = a[i * aCols + k];
+                        for (int j = jStart; j < n; j++)
+                        {
+                            c[cIdx++] += aValue * b[bIdx++];
+                        }
+                    }
+                }
+            }
+            private static unsafe void Kernel8x8(float* a, float* b, float* c, int p, int n, int aRows, int aCols, int bCols)
+            {
+                var cBackup = c;
+
+                var c_0_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_1_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_2_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_3_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_4_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_5_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_6_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_7_v = Avx2.LoadVector256(c);
+                c = cBackup;
+
+                Vector256<float> b_v;
+
+                for (int k = 0; k < p; k++)
+                {
+                    b_v = Avx2.LoadVector256(b);
+                    b += n;
+                    var ta = a + k;
+
+                    c_0_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_0_v);
+                    ta += aCols;
+                    c_1_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_1_v);
+                    ta += aCols;
+                    c_2_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_2_v);
+                    ta += aCols;
+                    c_3_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_3_v);
+                    ta += aCols;
+                    c_4_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_4_v);
+                    ta += aCols;
+                    c_5_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_5_v);
+                    ta += aCols;
+                    c_6_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_6_v);
+                    ta += aCols;
+                    c_7_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_7_v);
+                }
+
+                Avx2.Store(c, c_0_v);
+                c += bCols;
+                Avx2.Store(c, c_1_v);
+                c += bCols;
+                Avx2.Store(c, c_2_v);
+                c += bCols;
+                Avx2.Store(c, c_3_v);
+                c += bCols;
+                Avx2.Store(c, c_4_v);
+                c += bCols;
+                Avx2.Store(c, c_5_v);
+                c += bCols;
+                Avx2.Store(c, c_6_v);
+                c += bCols;
+                Avx2.Store(c, c_7_v);
+            }
+            private static unsafe void Kernel1x8(float* a, float* b, float* c, int p, int n, int aRows, int aCols, int bCols)
+            {
+                var c_0_v = Avx2.LoadVector256(c);
+
+                Vector256<float> b_v;
+
+                for (int k = 0; k < p; k++, b += n)
+                {
+                    b_v = Avx2.LoadVector256(b);
+
+                    c_0_v = Fma.MultiplyAdd(b_v, Vector256.Create(*a++), c_0_v);
+                }
+
+                Avx2.Store(c, c_0_v);
+            }
+        }
+
+        private static class Float64MicroKernel
+        {
+            public static unsafe void ExecBlock(double* a, double* b, double* c, int m, int p, int n, int aRows, int aCols, int bCols, double* packedB)
+            {
+                int aIdx, bIdx, cIdx;
+
+                for (int i = 0; i <= m - 8; i += 8)
+                {
+                    cIdx = i * bCols;
+                    aIdx = i * aCols;
+                    for (int j = 0; j <= n - 4; j += 4, cIdx += 4)
+                    {
+                        bIdx = j * p;
+                        if (i == 0) Utils.PackMatrixBWithPxQ(b + j, packedB + bIdx, p, 4, bCols);
+                        Kernel64b8x4(a + aIdx, packedB + bIdx, c + cIdx, p, 4, aRows, aCols, bCols);
+                    }
+                }
+                int iStart = m / 8 * 8;
+                aIdx = iStart * aCols;
+                bool hasPacked = m >= 8 && n >= 8;
+                for (int i = iStart; i < m; i++, aIdx += aCols)
+                {
+                    cIdx = i * bCols;
+                    for (int j = 0; j <= n - 4; j += 4, cIdx += 4)
+                    {
+                        if (!hasPacked && i == iStart) Utils.PackMatrixBWithPxQ(b + j, packedB + j * p, p, 4, bCols);
+                        Kernel64b1x4(a + aIdx, packedB + j * p, c + cIdx, p, 4, aRows, aCols, bCols);
+                    }
+                }
+                int jStart = n / 4 * 4;
+                double c0, c1, c2, c3;
+                for (int i = 0; i <= m - 4; i += 4)
+                {
+                    for (int j = jStart; j < n; j++)
+                    {
+                        int aIdx0 = i * aCols, aIdx1 = aIdx0 + aCols, aIdx2 = aIdx1 + aCols, aIdx3 = aIdx2 + aCols;
+                        cIdx = i * bCols + j;
+                        c0 = c[cIdx];
+                        cIdx += bCols;
+                        c1 = c[cIdx];
+                        cIdx += bCols;
+                        c2 = c[cIdx];
+                        cIdx += bCols;
+                        c3 = c[cIdx];
+
+                        for (int k = 0; k < p; k++)
+                        {
+                            var bValue = b[k * bCols + j];
+                            c0 += a[aIdx0++] * bValue;
+                            c1 += a[aIdx1++] * bValue;
+                            c2 += a[aIdx2++] * bValue;
+                            c3 += a[aIdx3++] * bValue;
+                        }
+                        cIdx = i * bCols + j;
+                        c[cIdx] = c0;
+                        cIdx += bCols;
+                        c[cIdx] = c1;
+                        cIdx += bCols;
+                        c[cIdx] = c2;
+                        cIdx += bCols;
+                        c[cIdx] = c3;
+                    }
+                }
+                for (int i = m / 4 * 4; i < m; i++)
+                {
+                    for (int k = 0; k < p; k++)
+                    {
+                        cIdx = i * bCols + jStart;
+                        bIdx = k * bCols + jStart;
+                        var aValue = a[i * aCols + k];
+                        for (int j = jStart; j < n; j++)
+                        {
+                            c[cIdx++] += aValue * b[bIdx++];
+                        }
                     }
                 }
             }
+            private static unsafe void Kernel64b8x4(double* a, double* b, double* c, int p, int n, int aRows, int aCols, int bCols)
+            {
+                var cBackup = c;
+
+                var c_0_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_1_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_2_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_3_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_4_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_5_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_6_v = Avx2.LoadVector256(c);
+                c += bCols;
+                var c_7_v = Avx2.LoadVector256(c);
+                c = cBackup;
+
+                Vector256<double> b_v;
+
+                for (int k = 0; k < p; k++)
+                {
+                    b_v = Avx2.LoadVector256(b);
+                    b += n;
+                    var ta = a + k;
+
+                    c_0_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_0_v);
+                    ta += aCols;
+                    c_1_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_1_v);
+                    ta += aCols;
+                    c_2_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_2_v);
+                    ta += aCols;
+                    c_3_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_3_v);
+                    ta += aCols;
+                    c_4_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_4_v);
+                    ta += aCols;
+                    c_5_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_5_v);
+                    ta += aCols;
+                    c_6_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_6_v);
+                    ta += aCols;
+                    c_7_v = Fma.MultiplyAdd(b_v, Vector256.Create(*ta), c_7_v);
+                }
+
+                Avx2.Store(c, c_0_v);
+                c += bCols;
+                Avx2.Store(c, c_1_v);
+                c += bCols;
+                Avx2.Store(c, c_2_v);
+                c += bCols;
+                Avx2.Store(c, c_3_v);
+                c += bCols;
+                Avx2.Store(c, c_4_v);
+                c += bCols;
+                Avx2.Store(c, c_5_v);
+                c += bCols;
+                Avx2.Store(c, c_6_v);
+                c += bCols;
+                Avx2.Store(c, c_7_v);
+            }
+            private static unsafe void Kernel64b1x4(double* a, double* b, double* c, int p, int n, int aRows, int aCols, int bCols)
+            {
+                var c_0_v = Avx2.LoadVector256(c);
+
+                Vector256<double> b_v;
+
+                for (int k = 0; k < p; k++, b += n)
+                {
+                    b_v = Avx2.LoadVector256(b);
+
+                    c_0_v = Fma.MultiplyAdd(b_v, Vector256.Create(*a++), c_0_v);
+                }
+
+                Avx2.Store(c, c_0_v);
+            }
         }
 
-        public static unsafe void Kernel32b8x8(ReadOnlySpan<T> a, ReadOnlySpan<T> b, Span<T> c, int aRows, int aCols, int bCols)
+        private static class GenericMicroKernel
         {
-            Vector<T> c_0_v = new Vector<T>();
-            Vector<T> c_1_v = new Vector<T>();
-            Vector<T> c_2_v = new Vector<T>();
-            Vector<T> c_3_v = new Vector<T>();
-            Vector<T> c_4_v = new Vector<T>();
-            Vector<T> c_5_v = new Vector<T>();
-            Vector<T> c_6_v = new Vector<T>();
-            Vector<T> c_7_v = new Vector<T>();
-
-            Vector<T> a_0_v, a_1_v, a_2_v, a_3_v, a_4_v, a_5_v, a_6_v, a_7_v;
-            Vector<T> b_v;
-            int offset = 0;
-
-            for(int k = 0; k < aCols; k++)
+            // (m, p) * (p, n)
+            public static unsafe void ExecBlock(T* a, T* b, T* c, int m, int p, int n, int aRows, int aCols, int bCols, T* packedB, int length)
+            {
+                int aIdx, bIdx, cIdx;
+
+                // 8 x <length> blocks
+                for (int i = 0; i <= m - 8; i += 8)
+                {
+                    cIdx = i * bCols;
+                    aIdx = i * aCols;
+                    for (int j = 0; j <= n - length; j += length, cIdx += length)
+                    {
+                        bIdx = j * p;
+                        if (i == 0) Utils.PackMatrixBWithPxQ(b + j, packedB + bIdx, p, length, bCols);
+                        Kernel8xL(a + aIdx, packedB + bIdx, c + cIdx, p, length, aRows, aCols, bCols, length);
+                    }
+                }
+                // The left rows and every <length> columns
+                int iStart = m / 8 * 8;
+                aIdx = iStart * aCols;
+                bool hasPacked = m >= 8 && n >= length;
+                for (int i = iStart; i < m; i++, aIdx += aCols)
+                {
+                    cIdx = i * bCols;
+                    for (int j = 0; j <= n - length; j += length, cIdx += length)
+                    {
+                        if (!hasPacked && i == iStart) Utils.PackMatrixBWithPxQ(b + j, packedB + j * p, p, length, bCols);
+                        Kernel1xL(a + aIdx, packedB + j * p, c + cIdx, p, length, aRows, aCols, bCols, length);
+                    }
+                }
+
+                // The left columns and every 4 rows
+                int jStart = n / length * length;
+                T c0, c1, c2, c3;
+                for (int i = 0; i <= m - 4; i += 4)
+                {
+                    for (int j = jStart; j < n; j++)
+                    {
+                        int aIdx0 = i * aCols, aIdx1 = aIdx0 + aCols, aIdx2 = aIdx1 + aCols, aIdx3 = aIdx2 + aCols;
+                        cIdx = i * bCols + j;
+                        c0 = c[cIdx];
+                        cIdx += bCols;
+                        c1 = c[cIdx];
+                        cIdx += bCols;
+                        c2 = c[cIdx];
+                        cIdx += bCols;
+                        c3 = c[cIdx];
+
+                        for (int k = 0; k < p; k++)
+                        {
+                            var bValue = b[k * bCols + j];
+#if NET7_0_OR_GREATER
+                            c0 += a[aIdx0++] * bValue;
+                            c1 += a[aIdx1++] * bValue;
+                            c2 += a[aIdx2++] * bValue;
+                            c3 += a[aIdx3++] * bValue;
+#else
+                            c0 = _handler.MultiplyAndAdd(a[aIdx0++], bValue, c0);
+                            c1 = _handler.MultiplyAndAdd(a[aIdx1++], bValue, c1);
+                            c2 = _handler.MultiplyAndAdd(a[aIdx2++], bValue, c2);
+                            c3 = _handler.MultiplyAndAdd(a[aIdx3++], bValue, c3);
+#endif
+                        }
+                        cIdx = i * bCols + j;
+                        c[cIdx] = c0;
+                        cIdx += bCols;
+                        c[cIdx] = c1;
+                        cIdx += bCols;
+                        c[cIdx] = c2;
+                        cIdx += bCols;
+                        c[cIdx] = c3;
+                    }
+                }
+                // The last 1-3 rows and the left columns
+                for (int i = m / 4 * 4; i < m; i++)
+                {
+                    for (int k = 0; k < p; k++)
+                    {
+                        cIdx = i * bCols + jStart;
+                        bIdx = k * bCols + jStart;
+                        var aValue = a[i * aCols + k];
+                        for (int j = jStart; j < n; j++)
+                        {
+#if NET7_0_OR_GREATER
+                            c[cIdx++] += aValue * b[bIdx++];
+#else
+                            var value = c[cIdx];
+                            c[cIdx++] = _handler.MultiplyAndAdd(aValue, b[bIdx++], value);
+#endif
+                        }
+                    }
+                }
+            }
+            private static unsafe void Kernel8xL(T* a, T* b, T* c, int p, int n, int aRows, int aCols, int bCols, int length)
+            {
+                var cBackup = c;
+
+#if NET7_0_OR_GREATER
+                var c_0_v = Vector256.Load(c);
+                c += bCols;
+                var c_1_v = Vector256.Load(c);
+                c += bCols;
+                var c_2_v = Vector256.Load(c);
+                c += bCols;
+                var c_3_v = Vector256.Load(c);
+                c += bCols;
+                var c_4_v = Vector256.Load(c);
+                c += bCols;
+                var c_5_v = Vector256.Load(c);
+                c += bCols;
+                var c_6_v = Vector256.Load(c);
+                c += bCols;
+                var c_7_v = Vector256.Load(c);
+                Vector256<T> b_v;
+#else
+                var c_0_v = new Vector<T>(new Span<T>(c, length));
+                c += bCols;
+                var c_1_v = new Vector<T>(new Span<T>(c, length));
+                c += bCols;
+                var c_2_v = new Vector<T>(new Span<T>(c, length));
+                c += bCols;
+                var c_3_v = new Vector<T>(new Span<T>(c, length));
+                c += bCols;
+                var c_4_v = new Vector<T>(new Span<T>(c, length));
+                c += bCols;
+                var c_5_v = new Vector<T>(new Span<T>(c, length));
+                c += bCols;
+                var c_6_v = new Vector<T>(new Span<T>(c, length));
+                c += bCols;
+                var c_7_v = new Vector<T>(new Span<T>(c, length));
+                Vector<T> b_v;
+#endif
+
+                c = cBackup;
+
+                for (int k = 0; k < p; k++)
+                {
+#if NET7_0_OR_GREATER
+                    b_v = Vector256.Load(b);
+#else
+                    b_v = new Vector<T>(new Span<T>(b, length));
+#endif
+                    b += n;
+                    var curA = a + k;
+
+                    c_0_v += b_v * *curA;
+                    curA += aCols;
+                    c_1_v += b_v * *curA;
+                    curA += aCols;
+                    c_2_v += b_v * *curA;
+                    curA += aCols;
+                    c_3_v += b_v * *curA;
+                    curA += aCols;
+                    c_4_v += b_v * *curA;
+                    curA += aCols;
+                    c_5_v += b_v * *curA;
+                    curA += aCols;
+                    c_6_v += b_v * *curA;
+                    curA += aCols;
+                    c_7_v += b_v * *curA;
+                }
+
+#if NET7_0_OR_GREATER
+                c_0_v.Store(c);
+                c += bCols;
+                c_1_v.Store(c);
+                c += bCols;
+                c_2_v.Store(c);
+                c += bCols;
+                c_3_v.Store(c);
+                c += bCols;
+                c_4_v.Store(c);
+                c += bCols;
+                c_5_v.Store(c);
+                c += bCols;
+                c_6_v.Store(c);
+                c += bCols;
+                c_7_v.Store(c);
+#else
+                c_0_v.CopyTo(new Span<T>(c, length));
+                c += bCols;
+                c_1_v.CopyTo(new Span<T>(c, length));
+                c += bCols;
+                c_2_v.CopyTo(new Span<T>(c, length));
+                c += bCols;
+                c_3_v.CopyTo(new Span<T>(c, length));
+                c += bCols;
+                c_4_v.CopyTo(new Span<T>(c, length));
+                c += bCols;
+                c_5_v.CopyTo(new Span<T>(c, length));
+                c += bCols;
+                c_6_v.CopyTo(new Span<T>(c, length));
+                c += bCols;
+                c_7_v.CopyTo(new Span<T>(c, length));
+#endif
+            }
+            private static unsafe void Kernel1xL(T* a, T* b, T* c, int p, int n, int aRows, int aCols, int bCols, int length)
             {
-                a_0_v = new Vector<T>(a[k]);
-                a_1_v = new Vector<T>(a[aCols + k]);
-                a_2_v = new Vector<T>(a[2 * aCols + k]);
-                a_3_v = new Vector<T>(a[3 * aCols + k]);
-                a_4_v = new Vector<T>(a[4 * aCols + k]);
-                a_5_v = new Vector<T>(a[5 * aCols + k]);
-                a_6_v = new Vector<T>(a[6 * aCols + k]);
-                a_7_v = new Vector<T>(a[7 * aCols + k]);
-
-                b_v = new Vector<T>(b.Slice(offset));
-                offset += bCols;
-
-                c_0_v += a_0_v * b_v;
-                c_1_v += a_1_v * b_v;
-                c_2_v += a_2_v * b_v;
-                c_3_v += a_3_v * b_v;
-                c_4_v += a_4_v * b_v;
-                c_5_v += a_5_v * b_v;
-                c_6_v += a_6_v * b_v;
-                c_7_v += a_7_v * b_v;
+#if NET7_0_OR_GREATER
+                var c_0_v = Vector256.Load(c);
+                Vector256<T> b_v;
+#else
+                var spanC = new Span<T>(c, length);
+                var c_0_v = new Vector<T>(spanC);
+                Vector<T> b_v;
+#endif
+
+                for (int k = 0; k < p; k++, b += n)
+                {
+#if NET7_0_OR_GREATER
+                    b_v = Vector256.Load(b);
+#else
+                    b_v = new Vector<T>(new Span<T>(b, length));
+#endif
+                    c_0_v += b_v * *a++;
+                }
+
+#if NET7_0_OR_GREATER
+                c_0_v.Store(c);
+#else
+                c_0_v.CopyTo(spanC);
+#endif
             }
-            c_0_v.CopyTo(c);
-            c_1_v.CopyTo(c.Slice(bCols));
-            c_2_v.CopyTo(c.Slice(2 * bCols));
-            c_3_v.CopyTo(c.Slice(3 * bCols));
-            c_4_v.CopyTo(c.Slice(4 * bCols));
-            c_5_v.CopyTo(c.Slice(5 * bCols));
-            c_6_v.CopyTo(c.Slice(6 * bCols));
-            c_7_v.CopyTo(c.Slice(7 * bCols));
         }
     }
 }
diff --git a/New Text Document.txt b/New Text Document.txt
new file mode 100644
index 0000000..c891023
--- /dev/null
+++ b/New Text Document.txt	
@@ -0,0 +1,6 @@
+0.671  19.67  15.93 （4x200）
+6.436   593.3  19.67
+
+50x1000: 423.3, 12.99, 23.06, 
+150x2000: 3416, 39.81, 40.71
+400x2000: 
\ No newline at end of file
diff --git a/iris.data b/iris.data
new file mode 100644
index 0000000..a3490e0
--- /dev/null
+++ b/iris.data
@@ -0,0 +1,150 @@
+5.1,3.5,1.4,0.2,Iris-setosa
+4.9,3.0,1.4,0.2,Iris-setosa
+4.7,3.2,1.3,0.2,Iris-setosa
+4.6,3.1,1.5,0.2,Iris-setosa
+5.0,3.6,1.4,0.2,Iris-setosa
+5.4,3.9,1.7,0.4,Iris-setosa
+4.6,3.4,1.4,0.3,Iris-setosa
+5.0,3.4,1.5,0.2,Iris-setosa
+4.4,2.9,1.4,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+5.4,3.7,1.5,0.2,Iris-setosa
+4.8,3.4,1.6,0.2,Iris-setosa
+4.8,3.0,1.4,0.1,Iris-setosa
+4.3,3.0,1.1,0.1,Iris-setosa
+5.8,4.0,1.2,0.2,Iris-setosa
+5.7,4.4,1.5,0.4,Iris-setosa
+5.4,3.9,1.3,0.4,Iris-setosa
+5.1,3.5,1.4,0.3,Iris-setosa
+5.7,3.8,1.7,0.3,Iris-setosa
+5.1,3.8,1.5,0.3,Iris-setosa
+5.4,3.4,1.7,0.2,Iris-setosa
+5.1,3.7,1.5,0.4,Iris-setosa
+4.6,3.6,1.0,0.2,Iris-setosa
+5.1,3.3,1.7,0.5,Iris-setosa
+4.8,3.4,1.9,0.2,Iris-setosa
+5.0,3.0,1.6,0.2,Iris-setosa
+5.0,3.4,1.6,0.4,Iris-setosa
+5.2,3.5,1.5,0.2,Iris-setosa
+5.2,3.4,1.4,0.2,Iris-setosa
+4.7,3.2,1.6,0.2,Iris-setosa
+4.8,3.1,1.6,0.2,Iris-setosa
+5.4,3.4,1.5,0.4,Iris-setosa
+5.2,4.1,1.5,0.1,Iris-setosa
+5.5,4.2,1.4,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+5.0,3.2,1.2,0.2,Iris-setosa
+5.5,3.5,1.3,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+4.4,3.0,1.3,0.2,Iris-setosa
+5.1,3.4,1.5,0.2,Iris-setosa
+5.0,3.5,1.3,0.3,Iris-setosa
+4.5,2.3,1.3,0.3,Iris-setosa
+4.4,3.2,1.3,0.2,Iris-setosa
+5.0,3.5,1.6,0.6,Iris-setosa
+5.1,3.8,1.9,0.4,Iris-setosa
+4.8,3.0,1.4,0.3,Iris-setosa
+5.1,3.8,1.6,0.2,Iris-setosa
+4.6,3.2,1.4,0.2,Iris-setosa
+5.3,3.7,1.5,0.2,Iris-setosa
+5.0,3.3,1.4,0.2,Iris-setosa
+7.0,3.2,4.7,1.4,Iris-versicolor
+6.4,3.2,4.5,1.5,Iris-versicolor
+6.9,3.1,4.9,1.5,Iris-versicolor
+5.5,2.3,4.0,1.3,Iris-versicolor
+6.5,2.8,4.6,1.5,Iris-versicolor
+5.7,2.8,4.5,1.3,Iris-versicolor
+6.3,3.3,4.7,1.6,Iris-versicolor
+4.9,2.4,3.3,1.0,Iris-versicolor
+6.6,2.9,4.6,1.3,Iris-versicolor
+5.2,2.7,3.9,1.4,Iris-versicolor
+5.0,2.0,3.5,1.0,Iris-versicolor
+5.9,3.0,4.2,1.5,Iris-versicolor
+6.0,2.2,4.0,1.0,Iris-versicolor
+6.1,2.9,4.7,1.4,Iris-versicolor
+5.6,2.9,3.6,1.3,Iris-versicolor
+6.7,3.1,4.4,1.4,Iris-versicolor
+5.6,3.0,4.5,1.5,Iris-versicolor
+5.8,2.7,4.1,1.0,Iris-versicolor
+6.2,2.2,4.5,1.5,Iris-versicolor
+5.6,2.5,3.9,1.1,Iris-versicolor
+5.9,3.2,4.8,1.8,Iris-versicolor
+6.1,2.8,4.0,1.3,Iris-versicolor
+6.3,2.5,4.9,1.5,Iris-versicolor
+6.1,2.8,4.7,1.2,Iris-versicolor
+6.4,2.9,4.3,1.3,Iris-versicolor
+6.6,3.0,4.4,1.4,Iris-versicolor
+6.8,2.8,4.8,1.4,Iris-versicolor
+6.7,3.0,5.0,1.7,Iris-versicolor
+6.0,2.9,4.5,1.5,Iris-versicolor
+5.7,2.6,3.5,1.0,Iris-versicolor
+5.5,2.4,3.8,1.1,Iris-versicolor
+5.5,2.4,3.7,1.0,Iris-versicolor
+5.8,2.7,3.9,1.2,Iris-versicolor
+6.0,2.7,5.1,1.6,Iris-versicolor
+5.4,3.0,4.5,1.5,Iris-versicolor
+6.0,3.4,4.5,1.6,Iris-versicolor
+6.7,3.1,4.7,1.5,Iris-versicolor
+6.3,2.3,4.4,1.3,Iris-versicolor
+5.6,3.0,4.1,1.3,Iris-versicolor
+5.5,2.5,4.0,1.3,Iris-versicolor
+5.5,2.6,4.4,1.2,Iris-versicolor
+6.1,3.0,4.6,1.4,Iris-versicolor
+5.8,2.6,4.0,1.2,Iris-versicolor
+5.0,2.3,3.3,1.0,Iris-versicolor
+5.6,2.7,4.2,1.3,Iris-versicolor
+5.7,3.0,4.2,1.2,Iris-versicolor
+5.7,2.9,4.2,1.3,Iris-versicolor
+6.2,2.9,4.3,1.3,Iris-versicolor
+5.1,2.5,3.0,1.1,Iris-versicolor
+5.7,2.8,4.1,1.3,Iris-versicolor
+6.3,3.3,6.0,2.5,Iris-virginica
+5.8,2.7,5.1,1.9,Iris-virginica
+7.1,3.0,5.9,2.1,Iris-virginica
+6.3,2.9,5.6,1.8,Iris-virginica
+6.5,3.0,5.8,2.2,Iris-virginica
+7.6,3.0,6.6,2.1,Iris-virginica
+4.9,2.5,4.5,1.7,Iris-virginica
+7.3,2.9,6.3,1.8,Iris-virginica
+6.7,2.5,5.8,1.8,Iris-virginica
+7.2,3.6,6.1,2.5,Iris-virginica
+6.5,3.2,5.1,2.0,Iris-virginica
+6.4,2.7,5.3,1.9,Iris-virginica
+6.8,3.0,5.5,2.1,Iris-virginica
+5.7,2.5,5.0,2.0,Iris-virginica
+5.8,2.8,5.1,2.4,Iris-virginica
+6.4,3.2,5.3,2.3,Iris-virginica
+6.5,3.0,5.5,1.8,Iris-virginica
+7.7,3.8,6.7,2.2,Iris-virginica
+7.7,2.6,6.9,2.3,Iris-virginica
+6.0,2.2,5.0,1.5,Iris-virginica
+6.9,3.2,5.7,2.3,Iris-virginica
+5.6,2.8,4.9,2.0,Iris-virginica
+7.7,2.8,6.7,2.0,Iris-virginica
+6.3,2.7,4.9,1.8,Iris-virginica
+6.7,3.3,5.7,2.1,Iris-virginica
+7.2,3.2,6.0,1.8,Iris-virginica
+6.2,2.8,4.8,1.8,Iris-virginica
+6.1,3.0,4.9,1.8,Iris-virginica
+6.4,2.8,5.6,2.1,Iris-virginica
+7.2,3.0,5.8,1.6,Iris-virginica
+7.4,2.8,6.1,1.9,Iris-virginica
+7.9,3.8,6.4,2.0,Iris-virginica
+6.4,2.8,5.6,2.2,Iris-virginica
+6.3,2.8,5.1,1.5,Iris-virginica
+6.1,2.6,5.6,1.4,Iris-virginica
+7.7,3.0,6.1,2.3,Iris-virginica
+6.3,3.4,5.6,2.4,Iris-virginica
+6.4,3.1,5.5,1.8,Iris-virginica
+6.0,3.0,4.8,1.8,Iris-virginica
+6.9,3.1,5.4,2.1,Iris-virginica
+6.7,3.1,5.6,2.4,Iris-virginica
+6.9,3.1,5.1,2.3,Iris-virginica
+5.8,2.7,5.1,1.9,Iris-virginica
+6.8,3.2,5.9,2.3,Iris-virginica
+6.7,3.3,5.7,2.5,Iris-virginica
+6.7,3.0,5.2,2.3,Iris-virginica
+6.3,2.5,5.0,1.9,Iris-virginica
+6.5,3.0,5.2,2.0,Iris-virginica
+6.2,3.4,5.4,2.3,Iris-virginica
+5.9,3.0,5.1,1.8,Iris-virginica
