-updated tests:

*updated build scripts
 *removed tls variables from code (needed to support WebAssembly)
 *some mmutils tweaks
 *some fixes
 *pthread TLS thread ID implementation
-added Atomic file (reimplementation of atomics templates for emscripten)
-added emscripten support to ecs.std
This commit is contained in:
Mergul 2019-11-25 20:06:16 +00:00
parent 46de0f6adb
commit 946fbf2934
18 changed files with 443 additions and 229 deletions

View file

@ -13,7 +13,7 @@ def compile(sources, output):
if file_extension == '.d' and filename != 'package':
files.append(os.path.join(r, file))
ldc_cmd = 'ldc2 ' + shared_flags + ldc_flags + '-oq -mtriple=wasm32-unknown-unknown-wasm -betterC --output-bc --od=.bc --singleobj --checkaction=C --of=' + output + ' '
ldc_cmd = compiler + shared_flags + ldc_flags + '-oq -mtriple=wasm32-unknown-unknown-wasm -betterC --output-bc --od=.bc --singleobj --checkaction=C --of=' + output + ' '
for path in sources:
ldc_cmd += '-I' + path + ' '
@ -31,12 +31,14 @@ def compile(sources, output):
print()
compiler = 'ldc2 '
#compiler = 'ldmd2 -vtls '
shared_flags = ''
clean = 0
demo = 0
sources = ['tests', 'source']
emc_flags = '-s USE_SDL=2 -s USE_SDL_IMAGE=2 -s SDL2_IMAGE_FORMATS="[\'png\']" --preload-file assets '
ldc_flags = '--d-version=SDL_209 --d-version=BindSDL_Static --d-version=BindSDL_Image --d-version=MM_USE_POSIX_THREADS '
ldc_flags = '--d-version=ECSEmscripten --d-version=SDL_209 --d-version=BindSDL_Static --d-version=BindSDL_Image --d-version=MM_USE_POSIX_THREADS '
import_paths = ['external/sources', 'external/imports', 'external/wasm_imports', '../source', 'utils/source', 'simple/source']
for arg in sys.argv[1:]:
@ -60,7 +62,7 @@ for arg in sys.argv[1:]:
shared_flags += '-g '
elif(arg == '-g4'):
ldc_flags += '-g '
emc_flags += '-g4 '
emc_flags += '-g4 --source-map-base ./ '
elif(arg == '--llvm-lto'):
emc_flags += '--llvm-lto 3 '
elif(arg == '--simd'):
@ -69,12 +71,12 @@ for arg in sys.argv[1:]:
shared_flags += '-O3 '
ldc_flags += '-release -enable-inlining '
emc_flags += '--llvm-lto 3 -s SIMD=1 '
elif(arg == '-quiet'):
emc_flags += "-Wl,--no-check-features "
elif(arg == '--clean'):
clean = 1
elif(arg == '-pthread'):
shared_flags += '-O3 '
ldc_flags += '-release -enable-inlining '
emc_flags += '-s PTHREAD_POOL_SIZE=16 -s USE_PTHREADS=1 '
emc_flags += '-s USE_PTHREADS=1 '
elif(arg == '--demo=simple'):
demo = 0
else:
@ -90,8 +92,8 @@ compile(['source'], 'demo.bc')
if clean or os.path.exists('../ecs.bc') == 0 or os.path.isfile('../ecs.bc') == 0:
compile(['../source'], '../ecs.bc')
emcc_cmd = 'emcc -v ' + shared_flags + emc_flags + '-s ERROR_ON_UNDEFINED_SYMBOLS=0 -s DISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR=1 -s TOTAL_MEMORY=256MB -s WASM_MEM_MAX=1024MB -s MALLOC=dlmalloc -s WASM=1 -o index.html '
#-s ALLOW_MEMORY_GROWTH=1 -s PROXY_TO_PTHREAD=1
emcc_cmd = 'emcc -v ' + shared_flags + emc_flags + '-s DISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR=1 -s ALLOW_MEMORY_GROWTH=1 -s WASM_MEM_MAX=2048MB -s MALLOC=dlmalloc -s WASM=1 -o index.html '
#-s ALLOW_MEMORY_GROWTH=1 -s PROXY_TO_PTHREAD=1 -Wl,--no-check-features -s ERROR_ON_UNDEFINED_SYMBOLS=0 -s TOTAL_MEMORY=512MB
emcc_cmd += '../ecs.bc '
emcc_cmd += 'utils.bc '

View file

@ -2,20 +2,20 @@ module glad.gl.funcs;
private import glad.gl.types;
bool GL_VERSION_1_0;
bool GL_VERSION_1_1;
bool GL_VERSION_1_2;
bool GL_VERSION_1_3;
bool GL_VERSION_1_4;
bool GL_VERSION_1_5;
bool GL_VERSION_2_0;
bool GL_VERSION_2_1;
bool GL_VERSION_3_0;
bool GL_VERSION_3_1;
bool GL_VERSION_3_2;
bool GL_VERSION_3_3;
bool GL_ES_VERSION_2_0;
bool GL_ES_VERSION_3_0;
__gshared bool GL_VERSION_1_0;
__gshared bool GL_VERSION_1_1;
__gshared bool GL_VERSION_1_2;
__gshared bool GL_VERSION_1_3;
__gshared bool GL_VERSION_1_4;
__gshared bool GL_VERSION_1_5;
__gshared bool GL_VERSION_2_0;
__gshared bool GL_VERSION_2_1;
__gshared bool GL_VERSION_3_0;
__gshared bool GL_VERSION_3_1;
__gshared bool GL_VERSION_3_2;
__gshared bool GL_VERSION_3_3;
__gshared bool GL_ES_VERSION_2_0;
__gshared bool GL_ES_VERSION_3_0;
nothrow @nogc extern(System) {
alias fp_glCullFace = void function(GLenum);
alias fp_glFrontFace = void function(GLenum);

View file

@ -109,7 +109,7 @@ bool gladLoadGL() {
return status;
}
static struct GLVersion { static int major = 0; static int minor = 0; }
__gshared struct GLVersion { __gshared int major = 0; __gshared int minor = 0; }
private extern(C) char* strstr(const(char)*, const(char)*) @nogc;
private extern(C) int strcmp(const(char)*, const(char)*) @nogc;
private extern(C) int strncmp(const(char)*, const(char)*, size_t) @nogc;

View file

@ -1,12 +1,11 @@
module mmutils.thread_pool;
import core.atomic;
import ecs.atomic;
//import core.stdc.stdio;
//import core.stdc.stdlib : free, malloc, realloc;
//import core.stdc.string : memcpy;
//import std.stdio;
import std.algorithm : map;
@ -108,6 +107,9 @@ version (WebAssembly)
}
extern(C) int clock_gettime(clockid_t, timespec*) @nogc nothrow @system;
extern(C) double emscripten_get_now() @nogc nothrow @system;
}
/// High precison timer
@ -122,13 +124,14 @@ long useconds()
return t.tv_sec * 1_000_000 + t.tv_usec;*/
time_t time;
timespec spec;
//time_t time;
//timespec spec;
clock_gettime(CLOCK_REALTIME, &spec);
//lock_gettime(CLOCK_REALTIME, &spec);
return cast(long)(emscripten_get_now() * 1000.0);
//time = spec.tv_sec;
return spec.tv_sec * 1000_000 + spec.tv_nsec / 1000;
//return spec.tv_sec * 1000_000 + spec.tv_nsec / 1000;
}
else version (Posix)
{
@ -190,6 +193,22 @@ void instructionPause()
static assert(0);
}
}
else version(WebAssembly)
{
version(LDC)
{
import ldc.attributes;
@optStrategy("none")
static void nop()
{
int i;
i++;
}
nop();
}
else static assert(0);
}
else static assert(0);
}
//////////////////////////////////////////////
@ -198,18 +217,17 @@ void instructionPause()
version (MM_USE_POSIX_THREADS)
{
version (Posix)
{
import core.sys.posix.pthread;
import core.sys.posix.semaphore;
}
else version (WebAssembly)
version (WebAssembly)
{
extern(C):
//alias uint time_t;
struct pthread_attr_t
{
union
{
int[10] __i;
uint[10] __s;
}
}
struct pthread_t
@ -230,7 +248,11 @@ version (MM_USE_POSIX_THREADS)
void pthread_exit(void *retval);
// semaphore.h
alias sem_t = void*;
//alias sem_t = void*;
struct sem_t
{
shared int[4] __val;
}
int sem_init(sem_t*, int, uint);
int sem_wait(sem_t*);
int sem_trywait(sem_t*);
@ -240,6 +262,11 @@ version (MM_USE_POSIX_THREADS)
//import core.sys.posix.pthread;
//import core.sys.posix.semaphore;
}
else version (Posix)
{
import core.sys.posix.pthread;
import core.sys.posix.semaphore;
}
else version (Windows)
{
extern (C):
@ -317,7 +344,7 @@ version (MM_USE_POSIX_THREADS)
void post()
{
int ret = sem_post(&mutex);
assert(ret == 0);
assert(ret >= 0);
}
void destroy()
@ -362,6 +389,9 @@ else version(D_BetterC)
{
version(Posix)
{
import core.sys.posix.pthread;
import core.sys.posix.semaphore;
struct Semaphore
{
sem_t mutex;
@ -619,7 +649,7 @@ else
///////////////// ThreadPool /////////////////
//////////////////////////////////////////////
private enum gMaxThreadsNum = 128;
private enum gMaxThreadsNum = 32;
alias JobDelegate = void delegate(ThreadData*, JobData*);
@ -634,7 +664,7 @@ struct JobLog
/// First in first out queue with atomic lock
struct JobQueue
{
alias LockType = long;
alias LockType = int;
align(64) shared LockType lock; /// Lock for accesing list of Jobs
align(64) JobData* first; /// Fist element in list of Jobs
@ -849,7 +879,8 @@ public:
threadData.threadPool = &this;
threadData.semaphore.initialize();
threadData.externalThread = true;
threadData.acceptJobs = true;
atomicStore(threadData.acceptJobs, true);
//threadData.acceptJobs = true;
int threadNum = atomicOp!"+="(threadsNum, 1) - 1;
@ -866,18 +897,19 @@ public:
void unregistExternalThread(ThreadData* threadData)
{
lockThreadsData();
scope (exit)
unlockThreadsData();
//scope (exit)
// unlockThreadsData();
disposeThreadData(threadData);
unlockThreadsData();
}
/// Allows external threads to return from threadStartFunc
void releaseExternalThreads()
{
lockThreadsData();
scope (exit)
unlockThreadsData();
//scope (exit)
// unlockThreadsData();
// Release external threads (including main thread)
foreach (i, ref ThreadData* th; threadsData)
@ -891,14 +923,15 @@ public:
addJobsRange(rng, cast(int) i);
atomicStore(th.end, true);
}
unlockThreadsData();
}
/// Waits for all threads to finish and joins them (excluding external threads)
void waitThreads()
{
lockThreadsData();
scope (exit)
unlockThreadsData();
//scope (exit)
// unlockThreadsData();
foreach (i, ref ThreadData* th; threadsData)
{
if (th is null)
@ -915,6 +948,7 @@ public:
th.thread.join();
disposeThreadData(th);
}
unlockThreadsData();
}
/// Sets number of threads to accept new jobs
@ -927,8 +961,8 @@ public:
assert(num > 0);
lockThreadsData();
scope (exit)
unlockThreadsData();
//scope (exit)
// unlockThreadsData();
foreach (i, ref ThreadData* th; threadsData)
{
@ -947,14 +981,15 @@ public:
th = makeThreadData();
th.threadPool = &this;
th.threadId = cast(int) i;
th.acceptJobs = true;
atomicStore(th.acceptJobs, true);
//th.acceptJobs = true;
th.semaphore.initialize();
th.thread.start(&th.threadStartFunc);
}
atomicStore(threadsNum, num);
unlockThreadsData();
}
/// Adds job to be executed by thread pool, such a job won't be synchronized with any group or job
@ -1012,8 +1047,9 @@ public:
{
assert(rng[0].group == threadData.group);
}
atomicOp!"+="(rng[0].group.jobsToBeDoneCount, cast(int) rng.length);
int threadsNumLocal = threadsNum;
int threadsNumLocal = atomicLoad(threadsNum);
int part = cast(int) rng.length / threadsNumLocal;
if (part > 0)
{
@ -1040,13 +1076,13 @@ public:
void addGroupAsynchronous(JobsGroup* group)
{
group.thPool = &this;
if (group.jobs.length == 0)
{
// Immediately call group end
group.onGroupFinish();
return;
}
group.setUpJobs();
auto rng = group.jobs[].map!((ref a) => &a);
addJobsRange(rng, group.executeOnThreadNum);
@ -1066,8 +1102,8 @@ public:
void flushAllLogs()
{
lockThreadsData();
scope (exit)
unlockThreadsData();
//scope (exit)
// unlockThreadsData();
foreach (thNum; 0 .. atomicLoad(threadsNum))
{
ThreadData* th = threadsData[thNum];
@ -1081,6 +1117,7 @@ public:
onThreadFlushLogs(th);
}
unlockThreadsData();
}
/// Default implementation of flushing logs
@ -1161,7 +1198,7 @@ private:
foreach (i; 0 .. 1_000)
{
if (threadNum >= threadsNum)
if (threadNum >= atomicLoad(threadsNum))
{
threadNum = 0;
atomicStore(threadSelector, 0);
@ -1287,7 +1324,10 @@ public:
this.name = name;
this.jobs = jobs;
this.executeOnThreadNum = executeOnThreadNum;
jobsToBeDoneCount = 0;
//jobsToBeDoneCount = 0;
//dependenciesWaitCount = 0;
atomicStore(jobsToBeDoneCount,0);
atomicStore(dependenciesWaitCount,0);
}
~this() nothrow
@ -1330,6 +1370,7 @@ private:
if (spawnedByGroup)
{
auto num = atomicOp!"-="(spawnedByGroup.jobsToBeDoneCount, 1);
assert(num >= 0);
if (num == 0)
{
spawnedByGroup.onGroupFinish();
@ -1438,7 +1479,8 @@ private void threadFunc(ThreadData* threadData)
}
}
threadData.end = false;
//threadData.end = false;
atomicStore(threadData.end, false);
assert(threadData.jobsQueue.empty());
}

View file

@ -160,14 +160,14 @@ struct CleanSystem
void mainLoop(void* arg)
{
static double time = 0;
__gshared double time = 0;
launcher.delta_time = launcher.getTime() - time;
time = launcher.getTime();
if(launcher.delta_time > 1000)launcher.delta_time = 1000;
static uint temp_fps = 0;
static double fps_time = 0;
__gshared uint temp_fps = 0;
__gshared double fps_time = 0;
temp_fps++;
fps_time += launcher.delta_time;
while(fps_time > 1000)
@ -295,7 +295,7 @@ void mainLoop(void* arg)
{
launcher.setStyle(2);
}
else if(igMenuItemBool("Super",null,launcher.style == 3,true))
else if(igMenuItemBool("Default",null,launcher.style == 3,true))
{
launcher.setStyle(3);
}
@ -466,8 +466,8 @@ void mainLoop(void* arg)
launcher.renderer.present();
draw_time = launcher.getTime() - draw_time;
static float plot_time = 0;
static uint plot_samples = 0;
__gshared float plot_time = 0;
__gshared uint plot_samples = 0;
plot_time += launcher.delta_time;
plot_samples++;
launcher.plot_values[100].delta_time += launcher.delta_time;
@ -602,14 +602,15 @@ int main(int argc, char** argv)
setStyle(3);
launcher.job_updater = Mallocator.make!ECSJobUpdater(8);
launcher.job_updater = Mallocator.make!ECSJobUpdater(12);
//launcher.job_updater.onCreate();
EntityManager.initialize(8);
EntityManager.initialize(12);
launcher.manager = EntityManager.instance;
launcher.manager.m_thread_id_func = &launcher.job_updater.getThreadID;
launcher.manager.setJobDispachFunc(&launcher.job_updater.dispatch);
//launcher.manager.m_thread_id_func = &launcher.job_updater.getThreadID;
//launcher.manager.setJobDispachFunc(&launcher.job_updater.dispatch);
launcher.manager.setMultithreadingCallbacks(&launcher.job_updater.dispatch, &launcher.job_updater.getThreadID);
launcher.manager.beginRegister();
@ -645,7 +646,7 @@ int main(int argc, char** argv)
//int result = emscripten_request_fullscreen_strategy(null, true, &strategy);
//emscripten_enter_soft_fullscreen(null, &strategy);
//printf("Fullscreen request: %s.\n", emscripten_result_to_string(result));
emscripten_set_main_loop_arg(&mainLoop, null, -1, 1);
emscripten_set_main_loop_arg(&mainLoop, null, 0, 1);
}
else
{

View file

@ -86,7 +86,7 @@ struct MoveSystem
}
}
Simple* simple;
__gshared Simple* simple;
void simpleStart()
{
@ -149,7 +149,7 @@ bool simpleLoop()
{
if(launcher.getKeyState(SDL_SCANCODE_SPACE))
{
spawnEntity();
foreach(i;0..10000)spawnEntity();
}
@ -159,7 +159,6 @@ bool simpleLoop()
launcher.job_updater.begin();
launcher.manager.updateMT();
launcher.job_updater.call();
launcher.multithreading = false;
}
else
{

View file

@ -409,7 +409,7 @@ struct CleanSystem
}
}
Snake* snake;
__gshared Snake* snake;
void snakeStart()
{
@ -500,7 +500,7 @@ bool snakeLoop()
float delta_time = launcher.delta_time;
if(delta_time > 2000)delta_time = 2000;
static float time = 0;
__gshared float time = 0;
if(launcher.getKeyState(SDL_SCANCODE_SPACE))time += delta_time * 3;
else time += delta_time;

View file

@ -746,8 +746,6 @@ bool spaceInvadersLoop()
launcher.job_updater.begin();
launcher.manager.updateMT();
launcher.job_updater.call();
launcher.multithreading = false;
printf("Disable mt\n");
}
else
{

View file

@ -2,6 +2,7 @@ module game_core.job_updater;
import ecs.std;
import ecs.vector;
import ecs.atomic;
import ecs_utils.utils;
@ -9,6 +10,17 @@ import ecs_utils.utils;
import ecs.manager;
import mmutils.thread_pool;
version(LDC)
{
import ldc.attributes;
}
else
{
struct optStrategy {
string strategy;
}
}
//import supre.core.call_graph_generator;
struct ECSJobUpdater
@ -24,14 +36,21 @@ struct ECSJobUpdater
pool.waitThreads();
//pool.unregistExternalThread(thread_data);
if(jobs)Mallocator.dispose(jobs);
version(WebAssembly)pthread_key_delete(tls_key);
}
version(WebAssembly)
{
__gshared pthread_key_t tls_key;
}
else static uint thread_id = 0;
ThreadPool pool;
ThreadData* thread_data;
int job_id = 0;
int no_dep_count = 0;
static uint thread_id = 0;
//static uint thread_id = 0;
struct Group
{
@ -78,21 +97,25 @@ struct ECSJobUpdater
Group[] jobs;
Vector!(Group*) call_jobs;
Group last_job;
JobData[1] groupEndJobs;
//TrackData[32] trackers;
void onCreate(uint threads_count)
{
version(WebAssembly)pthread_key_create(&tls_key, null);
pool.initialize();
//thread_data = pool.registerExternalThread();
thread_data = pool.registerExternalThread();
pool.setThreadsNum(threads_count);
jobs = Mallocator.makeArray!Group(256);
}
uint getThreadID() nothrow
uint getThreadID() @nogc nothrow
{
return thread_id;
version(WebAssembly)return cast(int)pthread_getspecific(tls_key);
else return thread_id;
}
void begin()
@ -113,36 +136,44 @@ struct ECSJobUpdater
//foreach(ref tracker;trackers)tracker.clear();
}
@optStrategy("none")
void nop()
{
int i;
i++;
}
//@optStrategy("none")
void call()
{
if(last_job.group.getDependenciesWaitCount() == 0)return;
if(call_jobs.length == 0)return;
JobData[1] groupEndJobs;
groupEndJobs[0] = JobData(&releaseMainThread, "Stop Threads");
//JobData[1] groupEndJobs;
groupEndJobs[0] = JobData(&releaseMainThread, "Stop Threads", null, null);
last_job.group.jobs = groupEndJobs;
last_job.group.thPool = &pool;
last_job.group.executeOnThreadNum = 0;
foreach(job;call_jobs)
{
job.start();
}
ret = false;
while(!ret)
/*while(atomicLoad(ret) == 1)//!cas(&ret,0,1))
{
printf("SDL\n");
}
printf("NonSDL\n");
//thread_data.threadStartFunc();
nop();
version(WebAssembly)//emscripten_main_thread_process_queued_calls();
}//*/
int i = 10;
thread_data.threadStartFunc();
}
shared bool ret = false;
void releaseMainThread(ThreadData* th_data, JobData* data)
{
//pool.releaseExternalThreads();
ret = true;
//atomicStore(ret,0);
pool.releaseExternalThreads();
}
static struct JobCaller
@ -153,10 +184,27 @@ shared bool ret = false;
void callJob(ThreadData* th_data, JobData* data)
{
updater.thread_id = th_data.threadId;
uint job_id = updater.getThreadID();
//uint job_id = updater.getThreadID();
//updater.trackers[job_id].begin(id);
job.execute();
version(WebAssembly)
{
//updater.thread_id = th_data.threadId;
pthread_setspecific(tls_key, cast(void*)th_data.threadId);
if(th_data.threadId == 0)
{
emscripten_main_thread_process_queued_calls();
job.execute();
emscripten_main_thread_process_queued_calls();
}
else job.execute();
}
else
{
updater.thread_id = th_data.threadId;
job.execute();
}
//atomicOp!"-="(updater.jobs_count,1);
//updater.trackers[job_id].end();
}
}
@ -178,7 +226,7 @@ shared bool ret = false;
caller.id = index;
jobs[group.id].add(caller);
}
jobs[group.id].build(&pool);
uint deps = cast(uint)group.dependencies.length;

View file

@ -87,7 +87,7 @@ struct Texture
}
}
static bool function(ref Texture this_, const char[] path) __load;
__gshared bool function(ref Texture this_, const char[] path) __load;
struct Data
{

View file

@ -28,12 +28,12 @@ import cimgui.cimgui;
extern(C):
SDL_Window* g_Window;
ulong g_Time;
bool[3] g_MousePressed;
SDL_Cursor*[ImGuiMouseCursor_COUNT] g_MouseCursors;
char* g_ClipboardTextData;
GLuint g_FontTexture = 0;
__gshared SDL_Window* g_Window;
__gshared ulong g_Time;
__gshared bool[3] g_MousePressed;
__gshared SDL_Cursor*[ImGuiMouseCursor_COUNT] g_MouseCursors;
__gshared char* g_ClipboardTextData;
__gshared GLuint g_FontTexture = 0;
const (char)* ImGuiImplSDL2GetClipboardText(void*)
{
@ -280,7 +280,7 @@ static void ImGui_ImplSDL2_UpdateGamepads()
}
private long frequency;
__gshared private long frequency;
void ImGuiImplSDL2NewFrame(SDL_Window* window)
{

View file

@ -50,6 +50,12 @@ version(D_BetterC)
version(WebAssembly)
{
alias pthread_key_t = uint;
extern (C) int pthread_key_create(pthread_key_t *, void* function(void *)) @nogc nothrow;
extern (C) int pthread_key_delete(pthread_key_t) @nogc nothrow;
extern (C) void* pthread_getspecific(pthread_key_t) @nogc nothrow;
extern (C) int pthread_setspecific(pthread_key_t, const void *) @nogc nothrow;
enum EMSCRIPTEN_RESULT_SUCCESS = 0;
enum EMSCRIPTEN_RESULT_DEFERRED = 1;
@ -109,6 +115,9 @@ version(WebAssembly)
extern (C) void emscripten_cancel_main_loop();
extern (C) int emscripten_request_fullscreen_strategy(const char *target, bool deferUntilInEventHandler, const EmscriptenFullscreenStrategy *fullscreenStrategy);
extern (C) int emscripten_enter_soft_fullscreen(const char *target, const EmscriptenFullscreenStrategy *fullscreenStrategy);
extern (C) void emscripten_main_thread_process_queued_calls();
extern (C) void emscripten_pause_main_loop();
extern (C) void emscripten_resume_main_loop();
alias int time_t;
alias int clockid_t;