因为开发的软件对大数据实时存储和处理有较高的要求，在搜索资料的过程中，发现一位大神写的非托管大数据数组的类，应用效果非凡，这里记录一下
阅读目录 (Content)
- C# 申请一个大数组 (Use a large array in C#)
- UnmanagedArray
- 如何使用 (How to use)
- 快速读写 UnmanagedArray
- UnmanagedArrayHelper
- 如何使用
- 2015-08-25
- 用 StructLayout 和 MarshalAs 支持复杂的 struct
C# 申请一个大数组 (Use a large array in C#)
在 C# 里，有时候我需要能够申请一个很大的数组、使用之、然后立即释放其占用的内存。
Sometimes I need to allocate a large array, use it and then release its memory space immediately.
由于在 C# 里提供的 int[] array = new int[1000000]; 这样的数组，其内存释放很难由程序员完全控制，在申请一个大数组后，程序可能会变得很慢。
If I use something like int[] array = new int[1000000]; , it will be difficult to release its memory space by programmer and the app probably runs slower and slower.
特别是在 C#+OpenGL 编程中，我在使用 VAO/VBO 时十分需要设计一个非托管的数组，比如在 glBufferData 时我希望可以使用下面的 glBufferData：
Specially in C#+OpenGL routines when I'm using VAO/VBO, I need an unmanaged array for glBufferData:
/// <summary>
/// 设置当前VBO的数据。
/// </summary>
/// <param ></param>
/// <param ></param>
/// <param ></param>
public static void glBufferData(uint target, UnmanagedArrayBase data, uint usage)
{
GetDelegateFor<glBufferData>()((uint)target,
data.ByteLength, // 使用非托管数组
data.Header, // 使用非托管数组
(uint)usage);
}
// ...
// glBufferData的声明
private delegate void glBufferData(uint target, int size, IntPtr data, uint usage);
而在指定 VBO 的数据时，可能是 float、vec3 等等类型：
And the content in VBO can be float, vec3 and any other structs.
/// <summary>
/// 金字塔的posotion array.
/// </summary>
static vec3[] positions = new vec3[]
{
new vec3(0.0f, 1.0f, 0.0f),
new vec3(-1.0f, -1.0f, 1.0f),
// ...
new vec3(-1.0f, -1.0f, 1.0f),
};
// Create a vertex buffer for the vertex data.
{
uint[] ids = new uint[1];
GL.GenBuffers(1, ids); GL.BindBuffer(GL.GL_ARRAY_BUFFER, ids[0]);
// 使用vec3作为泛型的非托管数组的参数
UnmanagedArray<vec3> positionArray = new UnmanagedArray<vec3>(positions.Length);
for (int i = 0; i < positions.Length; i++)
{
// 使用this[i]这样的索引方式来读写非托管数组的元素
positionArray[i] = positions[i];
}
GL.BufferData(BufferDataTarget.ArrayBuffer, positionArray, BufferDataUsage.StaticDraw);
GL.VertexAttribPointer(positionLocation, 3, GL.GL_FLOAT, false, 0, IntPtr.Zero);
GL.EnableVertexAttribArray(positionLocation);
}
UnmanagedArray
所以我设计了这样一个非托管的数组类型：无 unsafe，可接收任何 struct 类型作为泛型参数，可随时释放内存。
So I designed this UnmangedArray
/// <summary>
/// 元素类型为sbyte, byte, char, short, ushort, int, uint, long, ulong, float, double, decimal, bool或其它struct的非托管数组。
/// <para>不能使用enum类型作为T。</para>
/// </summary>
/// <typeparam name="T">sbyte, byte, char, short, ushort, int, uint, long, ulong, float, double, decimal, bool或其它struct, 不能使用enum类型作为T。</typeparam>
public class UnmanagedArray<T> : UnmanagedArrayBase where T : struct
{
/// <summary>
///元素类型为sbyte, byte, char, short, ushort, int, uint, long, ulong, float, double, decimal, bool或其它struct的非托管数组。
/// </summary>
/// <param name="count"></param>
[MethodImpl(MethodImplOptions.Synchronized)]
public UnmanagedArray(int count)
: base(count, Marshal.SizeOf(typeof(T)))
{
}
/// <summary>
/// 获取或设置索引为<paramref name="index"/>的元素。
/// </summary>
/// <param name="index"></param>
/// <returns></returns>
public T this[int index]
{
get
{
if (index < 0 || index >= this.Count)
throw new IndexOutOfRangeException("index of UnmanagedArray is out of range");
var pItem = this.Header + (index * elementSize);
//var obj = Marshal.PtrToStructure(pItem, typeof(T));
//T result = (T)obj;
T result = Marshal.PtrToStructure<T>(pItem);// works in .net 4.5.1
return result;
}
set
{
if (index < 0 || index >= this.Count)
throw new IndexOutOfRangeException("index of UnmanagedArray is out of range");
var pItem = this.Header + (index * elementSize);
//Marshal.StructureToPtr(value, pItem, true);
Marshal.StructureToPtr<T>(value, pItem, true);// works in .net 4.5.1
}
}
/// <summary>
/// 按索引顺序依次获取各个元素。
/// </summary>
/// <returns></returns>
public IEnumerable<T> GetElements()
{
if (!this.disposed)
{
for (int i = 0; i < this.Count; i++)
{
yield return this[i];
}
}
}
}
/// <summary>
/// 非托管数组的基类。
/// </summary>
public abstract class UnmanagedArrayBase : IDisposable
{
/// <summary>
/// 数组指针。
/// </summary>
public IntPtr Header { get; private set; }
/// <summary>
/// 元素数目。
/// </summary>
public int Count { get; private set; }
/// <summary>
/// 单个元素的字节数。
/// </summary>
protected int elementSize;
/// <summary>
/// 申请到的字节数。（元素数目 * 单个元素的字节数）。
/// </summary>
public int ByteLength
{
get { return this.Count * this.elementSize; }
}
/// <summary>
/// 非托管数组。
/// </summary>
/// <param name="elementCount">元素数目。</param>
/// <param name="elementSize">单个元素的字节数。</param>
[MethodImpl(MethodImplOptions.Synchronized)]
protected UnmanagedArrayBase(int elementCount, int elementSize)
{
this.Count = elementCount;
this.elementSize = elementSize;
int memSize = elementCount * elementSize;
this.Header = Marshal.AllocHGlobal(memSize);
allocatedArrays.Add(this);
}
private static readonly List<IDisposable> allocatedArrays = new List<IDisposable>();
/// <summary>
/// 立即释放所有<see cref="UnmanagedArray"/>。
/// </summary>
[MethodImpl(MethodImplOptions.Synchronized)]
public static void FreeAll()
{
foreach (var item in allocatedArrays)
{
item.Dispose();
}
allocatedArrays.Clear();
}
~UnmanagedArrayBase()
{
Dispose();
}
#region IDisposable Members
/// <summary>
/// Internal variable which checks if Dispose has already been called
/// </summary>
protected Boolean disposed;
/// <summary>
/// Releases unmanaged and - optionally - managed resources
/// </summary>
/// <param name="disposing"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
protected void Dispose(Boolean disposing)
{
if (disposed)
{
return;
}
if (disposing)
{
//Managed cleanup code here, while managed refs still valid
}
//Unmanaged cleanup code here
IntPtr ptr = this.Header;
if (ptr != IntPtr.Zero)
{
this.Count = 0;
this.Header = IntPtr.Zero;
Marshal.FreeHGlobal(ptr);
}
disposed = true;
}
/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
public void Dispose()
{
this.Dispose(true);
GC.SuppressFinalize(this);
}
#endregion
}
如何使用 (How to use)
UnmanagedArray
Using UnamangedAray
internal static void TypicalScene()
{
const int count = 100;
// 测试float类型
var floatArray = new UnmanagedArray<float>(count);
for (int i = 0; i < count; i++)
{
floatArray[i] = i;
}
for (int i = 0; i < count; i++)
{
var item = floatArray[i];
if (item != i)
{ throw new Exception(); }
}
// 测试int类型
var intArray = new UnmanagedArray<int>(count);
for (int i = 0; i < count; i++)
{
intArray[i] = i;
}
for (int i = 0; i < count; i++)
{
var item = intArray[i];
if (item != i)
{ throw new Exception(); }
}
// 测试bool类型
var boolArray = new UnmanagedArray<bool>(count);
for (int i = 0; i < count; i++)
{
boolArray[i] = i % 2 == 0;
}
for (int i = 0; i < count; i++)
{
var item = boolArray[i];
if (item != (i % 2 == 0))
{ throw new Exception(); }
}
// 测试vec3类型
var vec3Array = new UnmanagedArray<vec3>(count);
for (int i = 0; i < count; i++)
{
vec3Array[i] = new vec3(i * 3 + 0, i * 3 + 1, i * 3 + 2);
}
for (int i = 0; i < count; i++)
{
var item = vec3Array[i];
var old = new vec3(i * 3 + 0, i * 3 + 1, i * 3 + 2);
if (item.x != old.x || item.y != old.y || item.z != old.z)
{ throw new Exception(); }
}
// 测试foreach
foreach (var item in vec3Array.GetElements())
{
Console.WriteLine(item);
}
// 释放此数组占用的内存，这之后就不能再使用vec3Array了。
vec3Array.Dispose();
// 立即释放所有非托管数组占用的内存，这之后就不能再使用上面申请的数组了。
UnmanagedArrayBase.FreeAll();
}快速读写 UnmanagedArray
UnmanagedArrayHelper
由于很多时候需要申请和使用很大的 UnmanagedArray
public static class UnmanagedArrayHelper
{
///// <summary>
///// 错误 1 无法获取托管类型(“T”)的地址和大小，或无法声明指向它的指针
///// </summary>
///// <typeparam name="T"></typeparam>
///// <param name="array"></param>
///// <returns></returns>
//public static unsafe T* FirstElement<T>(this UnmanagedArray<T> array) where T : struct
//{
// var header = (void*)array.Header;
// return (T*)header;
//}
/// <summary>
/// 获取非托管数组的第一个元素的地址。
/// </summary>
/// <param name="array"></param>
/// <returns></returns>
public static unsafe void* FirstElement(this UnmanagedArrayBase array)
{
var header = (void*)array.Header;
return header;
}
public static unsafe void* LastElement(this UnmanagedArrayBase array)
{
var last = (void*)(array.Header + (array.ByteLength - array.ByteLength / array.Length));
return last;
}
/// <summary>
/// 获取非托管数组的最后一个元素的地址再向后一个单位的地址。
/// </summary>
/// <param name="array"></param>
/// <returns></returns>
public static unsafe void* TailAddress(this UnmanagedArrayBase array)
{
var tail = (void*)(array.Header + array.ByteLength);
return tail;
}
}如何使用
这个类型实现了 3 个扩展方法，可以获取 UnmanagedArray
下面是一个例子。用 unsafe 的方式读写 UnmanagedArray
public static void TypicalScene()
{
int length = 1000000;
UnmanagedArray<int> array = new UnmanagedArray<int>(length);
UnmanagedArray<int> array2 = new UnmanagedArray<int>(length);
long tick = DateTime.Now.Ticks;
for (int i = 0; i < length; i++)
{
array[i] = i;
}
long totalTicks = DateTime.Now.Ticks - tick;
tick = DateTime.Now.Ticks;
unsafe
{
int* header = (int*)array2.FirstElement();
int* last = (int*)array2.LastElement();
int* tailAddress = (int*)array2.TailAddress();
int value = 0;
for (int* ptr = header; ptr <= last/*or: ptr < tailAddress*/; ptr++)
{
*ptr = value++;
}
}
long totalTicks2 = DateTime.Now.Ticks - tick;
Console.WriteLine("ticks: {0}, {1}", totalTicks, totalTicks2);// unsafe method works faster.
for (int i = 0; i < length; i++)
{
if (array[i] != i)
{
Console.WriteLine("something wrong here");
}
if (array2[i] != i)
{
Console.WriteLine("something wrong here");
}
}
array.Dispose();
array2.Dispose();
} unsafe
{
vec3* header = (vec3*)vec3Array.FirstElement();
vec3* last = (vec3*)vec3Array.LastElement();
vec3* tailAddress = (vec3*)vec3Array.TailAddress();
int i = 0;
for (vec3* ptr = header; ptr <= last/*or: ptr < tailAddress*/; ptr++)
{
*ptr = new vec3(i * 3 + 0, i * 3 + 1, i * 3 + 2);
i++;
}
i = 0;
for (vec3* ptr = header; ptr <= last/*or: ptr < tailAddress*/; ptr++, i++)
{
var item = *ptr;
var old = new vec3(i * 3 + 0, i * 3 + 1, i * 3 + 2);
if (item.x != old.x || item.y != old.y || item.z != old.z)
{ throw new Exception(); }
}
}2015-08-25
用 StructLayout 和 MarshalAs 支持复杂的 struct
在 OpenGL 中我需要用 UnmanagedArray
/// <summary>
/// Represents a 4x4 matrix.
/// </summary>
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi, Size = 4 * 4 * 4)]
public struct mat4
{
/// <summary>
/// Gets or sets the <see cref="vec4"/> column at the specified index.
/// </summary>
/// <value>
/// The <see cref="vec4"/> column.
/// </value>
/// <param name="column">The column index.</param>
/// <returns>The column at index <paramref name="column"/>.</returns>
public vec4 this[int column]
{
get { return cols[column]; }
set { cols[column] = value; }
}
/// <summary>
/// Gets or sets the element at <paramref name="column"/> and <paramref name="row"/>.
/// </summary>
/// <value>
/// The element at <paramref name="column"/> and <paramref name="row"/>.
/// </value>
/// <param name="column">The column index.</param>
/// <param name="row">The row index.</param>
/// <returns>
/// The element at <paramref name="column"/> and <paramref name="row"/>.
/// </returns>
public float this[int column, int row]
{
get { return cols[column][row]; }
set { cols[column][row] = value; }
}
/// <summary>
/// The columms of the matrix.
/// </summary>
[MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)]
private vec4[] cols;
}
/// <summary>
/// Represents a four dimensional vector.
/// </summary>
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi, Size = 4 * 4)]
public struct vec4
{
public float x;
public float y;
public float z;
public float w;
public float this[int index]
{
get
{
if (index == 0) return x;
else if (index == 1) return y;
else if (index == 2) return z;
else if (index == 3) return w;
else throw new Exception("Out of range.");
}
set
{
if (index == 0) x = value;
else if (index == 1) y = value;
else if (index == 2) z = value;
else if (index == 3) w = value;
else throw new Exception("Out of range.");
}
}
}
mat4注意：UnmanagedArray
下面是测试用例。
mat4 matrix = glm.scale(mat4.identity(), new vec3(2, 3, 4));
var size = Marshal.SizeOf(typeof(mat4));
size = Marshal.SizeOf(matrix);
UnmanagedArray<mat4> array = new UnmanagedArray<mat4>(1);
array[0] = matrix;
mat4 newMatirx = array[0]; // newMatrix should be equal to matrix
array.Dispose();如果 matrix 和 newMatrix 相等，就说明上述 Attribute 配置正确了。
版权属于：CraZyMoNk
本文链接：https://blog.peacharc.com/archives/505.html
转载时须注明出处及本声明
