Add Add HashMap with dependices from mutils

This commit is contained in:
mmcomando 2018-09-08 20:18:30 +02:00
parent 6217aec6be
commit 9abc36be1c
4 changed files with 971 additions and 0 deletions

396
source/ecs/hash_map.d Executable file
View file

@ -0,0 +1,396 @@
module ecs.hash_map;
import std.traits;
import ecs.vector;
import ecs.traits;
enum doNotInline = "version(DigitalMars)pragma(inline,false);version(LDC)pragma(LDC_never_inline);";
private enum HASH_EMPTY = 0;
private enum HASH_DELETED = 0x1;
private enum HASH_FILLED_MARK = ulong(1) << 8 * ulong.sizeof - 1;
ulong defaultHashFunc(T)(auto ref T t) {
static if (isIntegral!(T)) {
return hashInt(t);
} else {
return hashInt(t.hashOf); // hashOf is not giving proper distribution between H1 and H2 hash parts
}
}
// Can turn bad hash function to good one
ulong hashInt(ulong x) nothrow @nogc @safe {
x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
x = x ^ (x >> 31);
return x;
}
struct HashMap(KeyPar, ValuePar, alias hashFunc = defaultHashFunc) {
alias Key = KeyPar;
alias Value = ValuePar;
enum rehashFactor = 0.75;
enum size_t getIndexEmptyValue = size_t.max;
static struct KeyVal {
Key key;
Value value;
}
static struct Bucket {
ulong hash;
KeyVal keyValue;
}
Vector!Bucket elements; // Length should be always power of 2
size_t length; // Used to compute loadFactor
size_t markerdDeleted;
void clear() {
elements.clear();
length = 0;
markerdDeleted = 0;
}
void reset() {
elements.reset();
length = 0;
markerdDeleted = 0;
}
bool isIn(ref Key el) {
return getIndex(el) != getIndexEmptyValue;
}
bool isIn(Key el) {
return getIndex(el) != getIndexEmptyValue;
}
Value* getPtr()(auto ref Key k) {
size_t index = getIndex(k);
if (index == getIndexEmptyValue) {
return null;
} else {
return &elements[index].keyValue.value;
}
}
ref Value get()(auto ref Key k) {
size_t index = getIndex(k);
assert(index != getIndexEmptyValue);
return elements[index].keyValue.value;
}
deprecated("Use get with second parameter.") auto ref Value getDefault()(
auto ref Key k, auto ref Value defaultValue) {
return get(k, defaultValue);
}
auto ref Value get()(auto ref Key k, auto ref Value defaultValue) {
size_t index = getIndex(k);
if (index == getIndexEmptyValue) {
return defaultValue;
} else {
return elements[index].keyValue.value;
}
}
ref Value getInsertDefault()(auto ref Key k, auto ref Value defaultValue) {
size_t index = getIndex(k);
if (index == getIndexEmptyValue) {
add(k, defaultValue);
}
index = getIndex(k);
assert(index != getIndexEmptyValue);
return elements[index].keyValue.value;
}
bool tryRemove(Key el) {
size_t index = getIndex(el);
if (index == getIndexEmptyValue) {
return false;
}
length--;
elements[index].hash = HASH_DELETED;
markerdDeleted++;
return true;
}
void remove(Key el) {
bool ok = tryRemove(el);
assert(ok);
}
ref Value opIndex()(auto ref Key key) {
return get(key);
}
void opIndexAssign()(auto ref Value value, auto ref Key key) {
add(key, value);
}
void add()(auto ref Key key, auto ref Value value) {
size_t index = getIndex(key);
if (index != getIndexEmptyValue) {
elements[index].keyValue.value = value;
return;
}
if (getLoadFactor(length + 1) > rehashFactor
|| getLoadFactor(length + markerdDeleted) > rehashFactor) {
rehash();
}
length++;
immutable ulong hash = hashFunc(key) | HASH_FILLED_MARK;
immutable size_t rotateMask = elements.length - 1;
index = hash & rotateMask; // Starting point
while (true) {
Bucket* gr = &elements[index];
if ((gr.hash & HASH_FILLED_MARK) == 0) {
if (gr.hash == HASH_DELETED) {
markerdDeleted--;
}
gr.hash = hash;
gr.keyValue.key = key;
gr.keyValue.value = value;
return;
}
index++;
index = index & rotateMask;
}
}
// For debug
//int numA;
//int numB;
size_t getIndex(Key el) {
return getIndex(el);
}
size_t getIndex(ref Key el) {
mixin(doNotInline);
immutable size_t groupsLength = elements.length;
if (groupsLength == 0) {
return getIndexEmptyValue;
}
immutable ulong hash = hashFunc(el) | HASH_FILLED_MARK;
immutable size_t rotateMask = groupsLength - 1;
size_t index = hash & rotateMask; // Starting point
//numA++;
while (true) {
//numB++;
Bucket* gr = &elements[index];
if (gr.hash == hash && gr.keyValue.key == el) {
return index;
}
if (gr.hash == HASH_EMPTY) {
return getIndexEmptyValue;
}
index++;
index = index & rotateMask;
}
}
float getLoadFactor(size_t forElementsNum) {
if (elements.length == 0) {
return 1;
}
return cast(float) forElementsNum / (elements.length);
}
void rehash() {
mixin(doNotInline);
// Get all elements
Vector!KeyVal allElements;
allElements.reserve(elements.length);
foreach (ref Bucket el; elements) {
if ((el.hash & HASH_FILLED_MARK) == 0) {
el.hash = HASH_EMPTY;
continue;
}
el.hash = HASH_EMPTY;
allElements ~= el.keyValue;
}
if (getLoadFactor(length + 1) > rehashFactor) { // Reallocate
elements.length = (elements.length ? elements.length : 4) << 1; // Power of two, initially 8 elements
}
// Insert elements
foreach (i, ref el; allElements) {
add(el.key, el.value);
}
length = allElements.length;
markerdDeleted = 0;
}
// foreach support
int opApply(DG)(scope DG dg) {
int result;
foreach (ref Bucket gr; elements) {
if ((gr.hash & HASH_FILLED_MARK) == 0) {
continue;
}
static if (isForeachDelegateWithTypes!(DG, Key)) {
result = dg(gr.keyValue.key);
} else static if (isForeachDelegateWithTypes!(DG, Value)) {
result = dg(gr.keyValue.value);
} else static if (isForeachDelegateWithTypes!(DG, Key, Value)) {
result = dg(gr.keyValue.key, gr.keyValue.value);
} else {
static assert(0);
}
if (result)
break;
}
return result;
}
int byKey(scope int delegate(Key k) dg) {
int result;
foreach (ref Key k; this) {
result = dg(k);
if (result)
break;
}
return result;
}
int byValue(scope int delegate(ref Value k) dg) {
int result;
foreach (ref Value v; this) {
result = dg(v);
if (result)
break;
}
return result;
}
int byKeyValue(scope int delegate(ref Key k, ref Value v) dg) {
int result;
foreach (ref Key k, ref Value v; this) {
result = dg(k, v);
if (result)
break;
}
return result;
}
import std.format : FormatSpec, formatValue;
/**
* Preety print
*/
void toString(scope void delegate(const(char)[]) sink, FormatSpec!char fmt) {
formatValue(sink, '[', fmt);
foreach (ref k, ref v; &byKeyValue) {
formatValue(sink, k, fmt);
formatValue(sink, ':', fmt);
formatValue(sink, v, fmt);
formatValue(sink, ", ", fmt);
}
formatValue(sink, ']', fmt);
}
}
static void dumpHashMapToJson(T)(ref T map, string path = "HashMapDump.json") {
Vector!char data;
import std.file;
import mutils.serializer.json;
JSONSerializer.instance.serialize!(Load.no)(map, data);
std.file.write(path, data[]);
}
static void printHashMap(T)(ref T map) {
import std.stdio;
writeln(T.stringof, " dump:\n");
foreach (k, v; &map.byKeyValue) {
writefln("%20s : %20s", k, v);
}
}
unittest {
HashMap!(int, int) map;
assert(map.isIn(123) == false);
assert(map.markerdDeleted == 0);
map.add(123, 1);
map.add(123, 1);
assert(map.isIn(123) == true);
assert(map.isIn(122) == false);
assert(map.length == 1);
map.remove(123);
assert(map.markerdDeleted == 1);
assert(map.isIn(123) == false);
assert(map.length == 0);
assert(map.tryRemove(500) == false);
map.add(123, 1);
assert(map.markerdDeleted == 0);
assert(map.tryRemove(123) == true);
foreach (i; 1 .. 130) {
map.add(i, 1);
}
foreach (i; 1 .. 130) {
assert(map.isIn(i));
}
foreach (i; 130 .. 500) {
assert(!map.isIn(i));
}
foreach (int el; map) {
assert(map.isIn(el));
}
}
unittest {
HashMap!(int, int) map;
map.add(1, 10);
assert(map.get(1) == 10);
assert(map.get(2, 20) == 20);
assert(!map.isIn(2));
assert(map.getInsertDefault(2, 20) == 20);
assert(map.get(2) == 20);
map[5] = 50;
assert(map[5] == 50);
foreach (k; &map.byKey) {
}
foreach (k, v; &map.byKeyValue) {
}
foreach (v; &map.byValue) {
}
}
unittest {
HashMap!(Vector!char, int) map;
Vector!char vecA;
vecA ~= "AAA";
map.add(vecA, 10);
assert(map[vecA] == 10);
map.add(vecA, 20);
assert(map[vecA] == 20);
//assert(vecA=="AAA");
//assert(map["AAA"]==10);// TODO hashMap Vector!char and string
}

137
source/ecs/string_intern.d Normal file
View file

@ -0,0 +1,137 @@
module ecs.string_intern;
import ecs.hash_map;
import ecs.traits : isForeachDelegateWithI;
import std.experimental.allocator;
import std.experimental.allocator.mallocator;
import std.traits : Parameters;
private __gshared static HashMap!(const(char)[], StringIntern) gStringInterns;
struct StringIntern {
private const(char)* strPtr;
this(const(char)[] fromStr) {
opAssign(fromStr);
}
void reset() {
strPtr=null;
}
size_t length() {
if (strPtr is null) {
return 0;
}
return *cast(size_t*)(strPtr - 8);
}
const(char)[] str() {
if (strPtr is null) {
return null;
}
return strPtr[0 .. length];
}
const(char)[] cstr() {
if (strPtr is null) {
return "\0";
}
return strPtr[0 .. length + 1];
}
bool opEquals()(auto ref const StringIntern s) {
return strPtr == s.strPtr;
}
bool opEquals()(auto ref const(char[]) s) {
return str() == s;
}
void opAssign(const(char)[] fromStr) {
if (fromStr.length == 0) {
return;
}
StringIntern defaultValue;
StringIntern internedStr = gStringInterns.get(fromStr, defaultValue);
if (internedStr.length == 0) {
internedStr.strPtr = allocStr(fromStr).ptr;
gStringInterns.add(internedStr.str, internedStr);
}
strPtr = internedStr.strPtr;
}
const(char)[] opSlice() {
if (strPtr is null) {
return null;
}
return strPtr[0 .. length];
}
private const(char)[] allocStr(const(char)[] fromStr) {
char[] data = Mallocator.instance.makeArray!(char)(fromStr.length + size_t.sizeof + 1);
size_t* len = cast(size_t*) data.ptr;
*len = fromStr.length;
data[size_t.sizeof .. $ - 1] = fromStr;
data[$ - 1] = '\0';
return data[size_t.sizeof .. $ - 1];
}
}
unittest {
static assert(StringIntern.sizeof == size_t.sizeof);
const(char)[] chA = ['a', 'a'];
char[] chB = ['o', 't', 'h', 'e', 'r'];
const(char)[] chC = ['o', 't', 'h', 'e', 'r'];
string chD = "other";
StringIntern strA;
StringIntern strB = StringIntern("");
StringIntern strC = StringIntern("a");
StringIntern strD = "a";
StringIntern strE = "aa";
StringIntern strF = chA;
StringIntern strG = chB;
assert(strA == strB);
assert(strA != strC);
assert(strC == strD);
assert(strD != strE);
assert(strE == strF);
assert(strD.length == 1);
assert(strE.length == 2);
assert(strG.length == 5);
strA = "other";
assert(strA == "other");
assert(strA == chB);
assert(strA == chC);
assert(strA == chD);
assert(strA.str.ptr[strA.str.length] == '\0');
assert(strA.cstr[$ - 1] == '\0');
foreach (char c; strA) {
}
foreach (int i, char c; strA) {
}
foreach (ubyte i, char c; strA) {
}
foreach (c; strA) {
}
}
unittest {
import mutils.container.hash_map : HashMap;
HashMap!(StringIntern, StringIntern) map;
map.add(StringIntern("aaa"), StringIntern("bbb"));
map.add(StringIntern("aaa"), StringIntern("bbb"));
assert(map.length == 1);
assert(map.get(StringIntern("aaa")) == StringIntern("bbb"));
}

39
source/ecs/traits.d Normal file
View file

@ -0,0 +1,39 @@
module ecs.traits;
import std.traits;
bool isForeachDelegateWithI(DG)() {
return is(DG == delegate) && is(ReturnType!DG == int)
&& Parameters!DG.length == 2 && isIntegral!(Parameters!(DG)[0]);
}
unittest {
assert(isForeachDelegateWithI!(int delegate(int, double)));
assert(isForeachDelegateWithI!(int delegate(int, double) @nogc nothrow));
assert(!isForeachDelegateWithI!(int delegate(double, double)));
}
bool isForeachDelegateWithoutI(DG)() {
return is(DG == delegate) && is(ReturnType!DG == int) && Parameters!DG.length == 1;
}
unittest {
assert(isForeachDelegateWithoutI!(int delegate(int)));
assert(isForeachDelegateWithoutI!(int delegate(size_t) @nogc nothrow));
assert(!isForeachDelegateWithoutI!(void delegate(int)));
}
bool isForeachDelegateWithTypes(DG, Types...)() {
return is(DG == delegate) && is(ReturnType!DG == int) && is(Parameters!DG == Types);
}
unittest {
assert(isForeachDelegateWithTypes!(int delegate(int, int), int, int));
assert(isForeachDelegateWithTypes!(int delegate(ref int, ref int), int, int));
assert(!isForeachDelegateWithTypes!(int delegate(double), int, int));
}
auto assumeNoGC(T)(T t) if (isFunctionPointer!T || isDelegate!T) {
enum attrs = functionAttributes!T | FunctionAttribute.nogc;
return cast(SetFunctionAttributes!(T, functionLinkage!T, attrs)) t;
}

399
source/ecs/vector.d Normal file
View file

@ -0,0 +1,399 @@
module ecs.vector;
import core.bitop;
import core.stdc.stdlib : free, malloc;
import core.stdc.string : memcpy, memset;
import std.algorithm : swap;
import std.conv : emplace;
import std.traits : hasMember, isCopyable, TemplateOf, Unqual;
@nogc @safe nothrow pure size_t nextPow2(size_t num) {
return 1 << bsr(num) + 1;
}
__gshared size_t gVectorsCreated = 0;
__gshared size_t gVectorsDestroyed = 0;
struct Vector(T) {
T[] array;
size_t used;
public:
this()(T t) {
add(t);
}
this(X)(X[] t) if (is(Unqual!X == Unqual!T)) {
add(t);
}
static if (isCopyable!T) {
this(this) {
T[] tmp = array[0 .. used];
array = null;
used = 0;
add(tmp);
}
} else {
@disable this(this);
}
~this() {
clear();
}
void clear() {
removeAll();
}
void removeAll() {
if (array !is null) {
foreach (ref el; array[0 .. used]) {
destroy(el);
}
freeData(cast(void[]) array);
gVectorsDestroyed++;
}
array = null;
used = 0;
}
bool empty() {
return (used == 0);
}
size_t length() {
return used;
}
void length(size_t newLength) {
if (newLength > used) {
reserve(newLength);
foreach (ref el; array[used .. newLength]) {
emplace(&el);
}
} else {
foreach (ref el; array[newLength .. used]) {
destroy(el);
}
}
used = newLength;
}
void reset() {
used = 0;
}
void reserve(size_t numElements) {
if (numElements > array.length) {
extend(numElements);
}
}
size_t capacity() {
return array.length - used;
}
void extend(size_t newNumOfElements) {
auto oldArray = manualExtend(array, newNumOfElements);
if (oldArray !is null) {
freeData(oldArray);
}
}
@nogc void freeData(void[] data) {
// 0x0F probably invalid value for pointers and other types
memset(data.ptr, 0x0F, data.length); // Makes bugs show up xD
free(data.ptr);
}
static void[] manualExtend(ref T[] array, size_t newNumOfElements = 0) {
if (newNumOfElements == 0)
newNumOfElements = 2;
if (array.length == 0)
gVectorsCreated++;
T[] oldArray = array;
size_t oldSize = oldArray.length * T.sizeof;
size_t newSize = newNumOfElements * T.sizeof;
T* memory = cast(T*) malloc(newSize);
memcpy(cast(void*) memory, cast(void*) oldArray.ptr, oldSize);
array = memory[0 .. newNumOfElements];
return cast(void[]) oldArray;
}
Vector!T copy()() {
Vector!T duplicate;
duplicate.reserve(used);
duplicate ~= array[0 .. used];
return duplicate;
}
bool canAddWithoutRealloc(uint elemNum = 1) {
return used + elemNum <= array.length;
}
void add()(T t) {
if (used >= array.length) {
extend(nextPow2(used + 1));
}
emplace(&array[used], t);
used++;
}
/// Add element at given position moving others
void add()(T t, size_t pos) {
assert(pos <= used);
if (used >= array.length) {
extend(array.length * 2);
}
foreach_reverse (size_t i; pos .. used) {
swap(array[i + 1], array[i]);
}
emplace(&array[pos], t);
used++;
}
void add(X)(X[] t) if (is(Unqual!X == Unqual!T)) {
if (used + t.length > array.length) {
extend(nextPow2(used + t.length));
}
foreach (i; 0 .. t.length) {
emplace(&array[used + i], t[i]);
}
used += t.length;
}
void remove(size_t elemNum) {
destroy(array[elemNum]);
swap(array[elemNum], array[used - 1]);
used--;
}
void removeStable()(size_t elemNum) {
used--;
foreach (i; 0 .. used) {
array[i] = array[i + 1];
}
}
bool tryRemoveElement()(T elem) {
foreach (i, ref el; array[0 .. used]) {
if (el == elem) {
remove(i);
return true;
}
}
return false;
}
void removeElement()(T elem) {
bool ok = tryRemoveElement(elem);
assert(ok, "There is no such an element in vector");
}
ref T opIndex(size_t elemNum) {
assert(elemNum < used, "Range violation [index]");
return array.ptr[elemNum];
}
auto opSlice() {
return array.ptr[0 .. used];
}
T[] opSlice(size_t x, size_t y) {
assert(y <= used);
return array.ptr[x .. y];
}
size_t opDollar() {
return used;
}
void opAssign(X)(X[] slice) {
reset();
this ~= slice;
}
void opOpAssign(string op)(T obj) {
static assert(op == "~");
add(obj);
}
void opOpAssign(string op, X)(X[] obj) {
static assert(op == "~");
add(obj);
}
void opIndexAssign()(T obj, size_t elemNum) {
assert(elemNum < used, "Range viloation");
array[elemNum] = obj;
}
void opSliceAssign()(T[] obj, size_t a, size_t b) {
assert(b <= used && a <= b, "Range viloation");
array.ptr[a .. b] = obj;
}
bool opEquals()(auto ref const Vector!(T) r) const {
return used == r.used && array.ptr[0 .. used] == r.array.ptr[0 .. r.used];
}
size_t toHash() const nothrow @trusted {
return hashOf(cast(Unqual!(T)[]) array.ptr[0 .. used]);
}
import std.format : FormatSpec, formatValue;
/**
* Preety print
*/
void toString(scope void delegate(const(char)[]) sink, FormatSpec!char fmt) {
static if (__traits(compiles, formatValue(sink, array[0 .. used], fmt))) {
formatValue(sink, array[0 .. used], fmt);
}
}
}
// Helper to avoid GC
private T[n] s(T, size_t n)(auto ref T[n] array) pure nothrow @nogc @safe {
return array;
}
@nogc nothrow unittest {
Vector!int vec;
assert(vec.empty);
vec.add(0);
vec.add(1);
vec.add(2);
vec.add(3);
vec.add(4);
vec.add(5);
assert(vec.length == 6);
assert(vec[3] == 3);
assert(vec[5] == 5);
assert(vec[] == [0, 1, 2, 3, 4, 5].s);
assert(!vec.empty);
vec.remove(3);
assert(vec.length == 5);
assert(vec[] == [0, 1, 2, 5, 4].s); //unstable remove
}
@nogc nothrow unittest {
Vector!int vec;
assert(vec.empty);
vec ~= [0, 1, 2, 3, 4, 5].s;
assert(vec[] == [0, 1, 2, 3, 4, 5].s);
assert(vec.length == 6);
vec ~= 6;
assert(vec[] == [0, 1, 2, 3, 4, 5, 6].s);
}
@nogc nothrow unittest {
Vector!int vec;
vec ~= [0, 1, 2, 3, 4, 5].s;
vec[3] = 33;
assert(vec[3] == 33);
}
@nogc nothrow unittest {
Vector!char vec;
vec ~= "abcd";
assert(vec[] == cast(char[]) "abcd");
}
@nogc nothrow unittest {
Vector!int vec;
vec ~= [0, 1, 2, 3, 4, 5].s;
vec.length = 2;
assert(vec[] == [0, 1].s);
}
///////////////////////////////////////////
enum string checkVectorAllocations = `
//assert(gVectorsCreated==gVectorsDestroyed);
gVectorsCreated=gVectorsDestroyed=0;
scope(exit){if(gVectorsCreated!=gVectorsDestroyed){
import std.stdio : writefln;
writefln("created==destroyed %s==%s", gVectorsCreated, gVectorsDestroyed);
assert(gVectorsCreated==gVectorsDestroyed, "Vector memory leak");
}}
`;
unittest {
mixin(checkVectorAllocations);
Vector!int vecA = Vector!int([0, 1, 2, 3, 4, 5].s);
assert(vecA[] == [0, 1, 2, 3, 4, 5].s);
Vector!int vecB;
vecB = vecA;
assert(vecB[] == [0, 1, 2, 3, 4, 5].s);
assert(vecB.array.ptr != vecA.array.ptr);
assert(vecB.used == vecA.used);
Vector!int vecC = vecA;
assert(vecC[] == [0, 1, 2, 3, 4, 5].s);
assert(vecC.array.ptr != vecA.array.ptr);
assert(vecC.used == vecA.used);
Vector!int vecD = vecA.init;
}
unittest {
static int numInit = 0;
static int numDestroy = 0;
scope (exit) {
assert(numInit == numDestroy);
}
static struct CheckDestructor {
int num = 1;
this(this) {
numInit++;
}
this(int n) {
num = n;
numInit++;
}
~this() {
numDestroy++;
}
}
CheckDestructor[2] arr = [CheckDestructor(1), CheckDestructor(1)];
Vector!CheckDestructor vec;
vec ~= CheckDestructor(1);
vec ~= arr;
vec.remove(1);
}
unittest {
assert(gVectorsCreated == gVectorsDestroyed);
gVectorsCreated = 0;
gVectorsDestroyed = 0;
scope (exit) {
assert(gVectorsCreated == gVectorsDestroyed);
}
string strA = "aaa bbb";
string strB = "ccc";
Vector!(Vector!char) vecA = Vector!(Vector!char)(Vector!char(cast(char[]) strA));
assert(vecA[0] == Vector!char(cast(char[]) strA));
Vector!(Vector!char) vecB;
vecB = vecA;
assert(vecB[0] == Vector!char(cast(char[]) strA));
assert(vecA.array.ptr != vecB.array.ptr);
assert(vecB.used == vecA.used);
assert(vecB[0].array.ptr != vecA[0].array.ptr);
assert(vecB[0].used == vecA[0].used);
}
unittest {
static struct Test {
int num;
@disable this(this);
}
Vector!Test test;
}