fanasina/y_project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change setup neurons and add update learning rates

Browse Source
This commit is contained in:
fanasina
2024-03-22 10:15:21 +01:00
parent 19d7e03fa7
commit 3911456abc
7 changed files with 498 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
neuron_t/src/neuron_t/neuron_t.c
+217 -53
View File
@@ -39,8 +39,36 @@ void free_config_layers(config_layers *pconf){
free(pconf);
}
bool randomizeInitWeight=true;
#define GEN_NEURONS_F_(type)\
\
void do_not_update_learnig_rate_##type(neurons_##type *N){}\
\
void time_based_update_learning_rate_##type(neurons_##type *nr){\
nr->learning_rate=(nr->learning_rate)/(1+(nr->decay_rate)*(nr->iteration_step));\
}\
\
type power_##type(type b, size_t p){\
type ret_pow=1;\
for(size_t i=0;i<p;++i) ret_pow *=b;\
return ret_pow;\
}\
void step_based_update_learning_rate_##type(neurons_##type *nr){\
nr->learning_rate=(nr->initial_learning_rate)*power_##type((nr->decay_rate),(1+(nr->iteration_step))/(nr->drop_rate));\
}\
\
void setup_learning_rate_params_neurons_##type(neurons_##type *base,type initial_learning_rate, type decay_rate, size_t drop_rate, void (*update_learning_rate)(neurons_##type *)){\
while(base){\
base->initial_learning_rate = initial_learning_rate;\
base->learning_rate = initial_learning_rate;\
base->decay_rate = decay_rate;\
base->drop_rate = drop_rate;\
base->update_learning_rate = update_learning_rate;\
base = base->next_layer;\
}\
}\
\
void calc_net_neurons_##type(neurons_##type *nr){\
size_t contractNB= ((nr->weight_in)->dim)->size - ((nr->input)->dim)->size ;\
/*print_tensor_msg_##type((nr->weight_in)," weight_in calc");*/\
@@ -66,6 +94,9 @@ type funcalc_delta_hidden_out_##type (type net, type temp, type(*df_act)(type)){
return df_act(net)* temp;\
}\
void calc_delta_neurons_##type(neurons_##type *nr){\
(nr->update_learning_rate)(nr);\
/*printf("leraning rt :%f , step : %ld\n",nr->learning_rate,nr->iteration_step);\
*/++(nr->iteration_step);\
if(nr->next_layer == NULL){\
if(nr->nb_calc_thread < 2){\
for(size_t i = 0; i<(nr->net)->dim->rank; ++i)\
@@ -107,6 +138,24 @@ void calc_delta_neurons_##type(neurons_##type *nr){\
free_tensor_##type(temp_w_d);\
}\
}\
\
type func_only_weight_in_##type(type w0, type w1, type scalar){\
return w0 - scalar * w1;\
}\
void only_update_weight_neurons_##type(neurons_##type *nr){\
if(nr->nb_calc_thread < 2){\
for(size_t i = 0; i<(nr->weight_in)->dim->rank; ++i){\
/*(nr->weight_in)->x[i]= (nr->weight_in)->x[i] - nr->learning_rate *tmp_e_w->x[i] ;\
*/(nr->weight_in)->x[i]= (nr->weight_in)->x[i] - nr->learning_rate * (nr->weight_out)->x[i] ;\
}\
}else{\
update_6tensor_func_##type(nr->weight_in, nr->weight_out, \
func_only_weight_in_##type,\
nr->learning_rate,\
nr->nb_calc_thread);\
}\
}\
\
void update_weight_neurons_##type(neurons_##type *nr){\
nr->TensorProduct(&(nr->weight_out), nr->input, nr->delta_out, nr->nb_prod_thread);\
/*tensor_##type *tmp_e_w=NULL;\
@@ -115,11 +164,13 @@ void update_weight_neurons_##type(neurons_##type *nr){\
/*print_tensor_msg_##type(nr->delta_out," nr delta_out update wei");*/\
/*print_tensor_msg_##type(tmp_e_w," tmp_e_w update wei");*/\
\
for(size_t i = 0; i<(nr->weight_in)->dim->rank; ++i){\
/*(nr->weight_in)->x[i]= (nr->weight_in)->x[i] - nr->learning_rate *tmp_e_w->x[i] ;\
*/(nr->weight_in)->x[i]= (nr->weight_in)->x[i] - nr->learning_rate * (nr->weight_out)->x[i] ;\
only_update_weight_neurons_##type(nr);\
/*for(size_t i = 0; i<(nr->weight_in)->dim->rank; ++i){\
*//*(nr->weight_in)->x[i]= (nr->weight_in)->x[i] - nr->learning_rate *tmp_e_w->x[i] ;\
*/\
/*(nr->weight_in)->x[i]= (nr->weight_in)->x[i] - nr->learning_rate * (nr->weight_out)->x[i] ;\
}\
/*print_tensor_msg_##type(nr->weight_in," weight_in updated ");*/\
*//*print_tensor_msg_##type(nr->weight_in," weight_in updated ");*/\
/*free_tensor_##type(tmp_e_w);\
*/\
}\
@@ -153,7 +204,7 @@ void link_layers_##type(neurons_##type *nPrev, neurons_##type *nNext ){\
}\
\
\
void setup_networks_alloutputs_##type(neurons_##type **base_nr, size_t **array_dim_in_layers, size_t *sz_layers, size_t nb_layers){\
void setup_networks_alloutputs_##type(neurons_##type **base_nr, size_t **array_dim_in_layers, size_t *sz_layers, size_t nb_layers, bool randomize, type minR, type maxR, int randomRange){\
neurons_##type *tmp_l=NULL, *ttmp_l=NULL;\
for(size_t l=0; l<nb_layers; ++l){\
tmp_l = malloc(sizeof(neurons_##type)); \
@@ -163,6 +214,7 @@ void setup_networks_alloutputs_##type(neurons_##type **base_nr, size_t **array_d
ttmp_l->next_layer = tmp_l ;\
}\
tmp_l->id_layer= l;\
tmp_l->iteration_step= 0;\
tmp_l->input = NULL; \
tmp_l->net = NULL; \
tmp_l->output = NULL; \
@@ -171,6 +223,7 @@ void setup_networks_alloutputs_##type(neurons_##type **base_nr, size_t **array_d
tmp_l->weight_out = NULL; \
tmp_l->delta_out = NULL; \
tmp_l->bias = NULL; \
tmp_l->update_learning_rate = do_not_update_learnig_rate_##type; \
tmp_l->prev_layer = ttmp_l;\
tmp_l->next_layer = NULL;\
\
@@ -191,9 +244,9 @@ void setup_networks_alloutputs_##type(neurons_##type **base_nr, size_t **array_d
dimension *d_w_in; \
add_dimension(&d_w_in, (ttmp_l->input)->dim, ((ttmp_l->output)->dim)); \
ttmp_l->weight_in = CREATE_TENSOR_##type(d_w_in);\
for(size_t i=0;i<((ttmp_l->weight_in)->dim)->rank;++i) (ttmp_l->weight_in)->x[i]=0.5;\
/*init_random_x_##type(ttmp_l->weight_in,0,1,5000);\
*/\
if(randomize) init_random_x_##type(ttmp_l->weight_in,minR,maxR,randomRange);\
else\
for(size_t i=0;i<((ttmp_l->weight_in)->dim)->rank;++i) (ttmp_l->weight_in)->x[i]=(minR+maxR)/2;\
}\
if(l==nb_layers-1) {\
dimension *dim_out=init_copy_dim(array_dim_in_layers[l],sz_layers[l]);\
@@ -208,9 +261,81 @@ void setup_networks_alloutputs_##type(neurons_##type **base_nr, size_t **array_d
dimension *d_w_in; \
add_dimension(&d_w_in, (tmp_l->input)->dim, ((tmp_l->output)->dim)); \
tmp_l->weight_in = CREATE_TENSOR_##type(d_w_in);\
if(randomize) init_random_x_##type(tmp_l->weight_in,minR,maxR,randomRange);\
else\
for(size_t i=0;i<((tmp_l->weight_in)->dim)->rank;++i) (tmp_l->weight_in)->x[i]=(minR+maxR)/2;\
}\
\
}\
\
ttmp_l = tmp_l;\
\
\
\
}\
}\
\
\
void setup_networks_alloutputs_GLOBAL_rdm01_##type(neurons_##type **base_nr, size_t **array_dim_in_layers, size_t *sz_layers, size_t nb_layers){\
neurons_##type *tmp_l=NULL, *ttmp_l=NULL;\
for(size_t l=0; l<nb_layers; ++l){\
tmp_l = malloc(sizeof(neurons_##type)); \
if(l==0){\
*base_nr = tmp_l ;\
}else{\
ttmp_l->next_layer = tmp_l ;\
}\
tmp_l->id_layer= l;\
tmp_l->iteration_step= 0;\
tmp_l->input = NULL; \
tmp_l->net = NULL; \
tmp_l->output = NULL; \
tmp_l->target = NULL; \
tmp_l->weight_in = NULL; \
tmp_l->weight_out = NULL; \
tmp_l->delta_out = NULL; \
tmp_l->bias = NULL; \
tmp_l->update_learning_rate = do_not_update_learnig_rate_##type; \
tmp_l->prev_layer = ttmp_l;\
tmp_l->next_layer = NULL;\
\
if(ttmp_l != NULL){\
dimension *dim=init_copy_dim(array_dim_in_layers[l-1],sz_layers[l-1]);\
increment_dim_var(dim);\
tmp_l->input = CREATE_TENSOR_##type(dim);\
for(size_t i=0;i<((tmp_l->input)->dim)->rank;++i) (tmp_l->input)->x[i]=(type)l;\
\
link_layers_##type(ttmp_l,tmp_l);\
if(l>1 ){\
dimension *dim_out = (ttmp_l->output)->dim;\
for(size_t i=0;i<dim_out->rank; ++i) (ttmp_l->output)->x[i]=(type)(l-1);\
ttmp_l->net = CREATE_TENSOR_FROM_CPY_DIM_##type(dim_out);\
for(size_t i=0;i<dim_out->rank; ++i) (ttmp_l->net)->x[i]=(type)(l-1);\
ttmp_l->delta_out = CREATE_TENSOR_FROM_CPY_DIM_##type(dim_out); \
for(size_t i=0;i< dim_out->rank; ++i) (ttmp_l->delta_out)->x[i]=(type)(l-1);\
dimension *d_w_in; \
add_dimension(&d_w_in, (ttmp_l->input)->dim, ((ttmp_l->output)->dim)); \
ttmp_l->weight_in = CREATE_TENSOR_##type(d_w_in);\
if(randomizeInitWeight) init_random_x_##type(ttmp_l->weight_in,0,1,5000);\
else\
for(size_t i=0;i<((ttmp_l->weight_in)->dim)->rank;++i) (ttmp_l->weight_in)->x[i]=0.5;\
}\
if(l==nb_layers-1) {\
dimension *dim_out=init_copy_dim(array_dim_in_layers[l],sz_layers[l]);\
tmp_l->output = CREATE_TENSOR_##type(dim_out);\
for(size_t i=0;i<((tmp_l->output)->dim)->rank;++i) (tmp_l->output)->x[i]=(type)l;\
tmp_l->target = CREATE_TENSOR_FROM_CPY_DIM_##type(dim_out);\
for(size_t i=0;i<((tmp_l->target)->dim)->rank;++i) (tmp_l->target)->x[i]=(type)(l);\
tmp_l->net = CREATE_TENSOR_FROM_CPY_DIM_##type(dim_out);\
for(size_t i=0;i<((tmp_l->net)->dim)->rank;++i) (tmp_l->net)->x[i]=(type)(l);\
tmp_l->delta_out = CREATE_TENSOR_FROM_CPY_DIM_##type(dim_out); \
for(size_t i=0;i<((tmp_l->delta_out)->dim)->rank;++i) (tmp_l->delta_out)->x[i]=(type)(l);\
dimension *d_w_in; \
add_dimension(&d_w_in, (tmp_l->input)->dim, ((tmp_l->output)->dim)); \
tmp_l->weight_in = CREATE_TENSOR_##type(d_w_in);\
if(randomizeInitWeight) init_random_x_##type(tmp_l->weight_in,0,1,5000);\
else\
for(size_t i=0;i<((tmp_l->weight_in)->dim)->rank;++i) (tmp_l->weight_in)->x[i]=0.5;\
/*init_random_x_##type(tmp_l->weight_in,0,1,5000);\
*/\
}\
\
}\
@@ -233,6 +358,7 @@ void setup_networks_layers_without_weights_##type(neurons_##type **base_nr, size
ttmp_l->next_layer = tmp_l ;\
}\
tmp_l->id_layer= l;\
tmp_l->iteration_step= 0;\
tmp_l->input = NULL; \
tmp_l->net = NULL; \
tmp_l->output = NULL; \
@@ -241,6 +367,7 @@ void setup_networks_layers_without_weights_##type(neurons_##type **base_nr, size
tmp_l->weight_out = NULL; \
tmp_l->delta_out = NULL; \
tmp_l->bias = NULL; \
tmp_l->update_learning_rate = do_not_update_learnig_rate_##type; \
tmp_l->prev_layer = ttmp_l;\
tmp_l->next_layer = NULL;\
\
@@ -309,8 +436,8 @@ void setup_weights_neurons_##type(neurons_##type *base, bool randomize, type min
\
}\
\
void setup_networks_alloutputs_config_##type(neurons_##type **base_nr, config_layers *lconf){\
setup_networks_alloutputs_##type(base_nr, lconf->array_dim_in_layers, lconf->sz_layers, lconf->nb_layers);\
void setup_networks_alloutputs_config_##type(neurons_##type **base_nr, config_layers *lconf, bool randomize, type minR, type maxR, int randomRange){\
setup_networks_alloutputs_##type(base_nr, lconf->array_dim_in_layers, lconf->sz_layers, lconf->nb_layers, randomize, minR, maxR, randomRange);\
}\
\
void setup_all_layers_functions_##type(neurons_##type *base, \
@@ -342,13 +469,14 @@ void setup_all_layers_params_##type(neurons_##type *base,\
while(temp){\
temp->nb_prod_thread=nb_prod_thread;\
temp->nb_calc_thread=nb_calc_thread;\
temp->initial_learning_rate=learning_rate;\
temp->learning_rate=learning_rate;\
temp=temp->next_layer;\
}\
}\
\
\
void setup_networks_OneD_##type(neurons_##type **base_nr, size_t *array_dim_in_layers, size_t nb_layers){\
void setup_networks_OneD_##type(neurons_##type **base_nr, size_t *array_dim_in_layers, size_t nb_layers, bool randomize, type minR, type maxR, int randomRange){\
size_t *sz_layers=malloc(nb_layers*sizeof(size_t));\
for(size_t i=0; i<nb_layers;++i) sz_layers[i]=1;\
size_t **tarray_dim_in_layers=malloc(nb_layers*sizeof(size_t));\
@@ -356,7 +484,7 @@ void setup_networks_OneD_##type(neurons_##type **base_nr, size_t *array_dim_in_l
tarray_dim_in_layers[i]=malloc(sizeof(size_t));\
tarray_dim_in_layers[i][0]=array_dim_in_layers[i];\
}\
setup_networks_alloutputs_##type(base_nr, tarray_dim_in_layers, sz_layers, nb_layers);\
setup_networks_alloutputs_##type(base_nr, tarray_dim_in_layers, sz_layers, nb_layers, randomize, minR, maxR, randomRange);\
\
for(size_t i=0; i<nb_layers;++i) {\
free(tarray_dim_in_layers[i]);\
@@ -420,7 +548,9 @@ void free_cloneuronset_##type(cloneuronset_##type *clnrnst){\
unlink_all_weigth_in_neurons_##type(clnrnst->cloneurons[i]);\
free_neurons_##type(clnrnst->cloneurons[i]);\
}\
free(clnrnst->cloneurons);\
free_config_layers(clnrnst->conf);\
free(clnrnst);\
}\
void link_cloneuronset_weight_in_funcs_params_from_base_##type(cloneuronset_##type *clnrnst){\
neurons_##type **tmp_c=malloc(clnrnst->nb_clone * sizeof(neurons_##type *));\
@@ -437,7 +567,11 @@ void link_cloneuronset_weight_in_funcs_params_from_base_##type(cloneuronset_##ty
tmp_c[c]->d_f_act = tmp_b->d_f_act;\
tmp_c[c]->TensorContraction = tmp_b->TensorContraction;\
tmp_c[c]->TensorProduct = tmp_b->TensorProduct;\
tmp_c[c]->initial_learning_rate = tmp_b->initial_learning_rate;\
tmp_c[c]->learning_rate = tmp_b->learning_rate;\
tmp_c[c]->decay_rate= tmp_b->decay_rate;\
tmp_c[c]->drop_rate= tmp_b->drop_rate;\
tmp_c[c]->update_learning_rate = tmp_b->update_learning_rate ;\
tmp_c[c]->nb_prod_thread = tmp_b->nb_prod_thread;\
tmp_c[c]->nb_calc_thread = tmp_b->nb_calc_thread;\
tmp_c[c]->id_layer = tmp_b->id_layer;\
@@ -486,6 +620,8 @@ void print_neurons_msg_##type(neurons_##type *nr, char *msg){\
PR_LINE;\
if(nr->target) print_tensor_msg_##type(nr->target," target "); else printf(" target NULL\n");\
PR_LINE;\
if(nr->next_layer == NULL) printf("next layer of %s = NULL\n",msg);\
PR_LINE;\
\
nr=nr->next_layer;\
}\
@@ -569,8 +705,8 @@ size_t learning_online_neurons_##type(neurons_##type *base, data_set_##type *dat
ttmp = ttmp->prev_layer;\
}\
}\
\
}while(!condition(error_out_##type(base), nbreps++));\
nbreps += (dataset->size);\
}while(!condition(error_out_##type(base), nbreps));\
\
\
printf(" ### reps : %ld \n",nbreps);\
@@ -592,11 +728,16 @@ size_t learning_online2_neurons_##type(neurons_##type *base, data_set_##type *da
}\
while(ttmp != base){\
calc_delta_neurons_##type(ttmp);\
update_weight_neurons_##type(ttmp);\
ttmp = ttmp->prev_layer;\
/*update_weight_neurons_##type(ttmp);\
*/ttmp = ttmp->prev_layer;\
}\
tmp = ttmp->next_layer;\
while(tmp){\
update_weight_neurons_##type(tmp);\
tmp = tmp->next_layer;\
}\
err = ABSMAX(err,error_out_##type(base));\
ending = condition(err, nbreps++);\
ending = condition(err, ++nbreps);\
}\
\
}while(!ending);\
@@ -640,13 +781,6 @@ void print_predict_by_network_with_error_neurons_##type(neurons_##type *base, te
\
}\
\
void only_update_weight_neurons_##type(neurons_##type *nr){\
for(size_t i = 0; i<(nr->weight_in)->dim->rank; ++i){\
/*(nr->weight_in)->x[i]= (nr->weight_in)->x[i] - nr->learning_rate *tmp_e_w->x[i] ;\
*/(nr->weight_in)->x[i]= (nr->weight_in)->x[i] - nr->learning_rate * (nr->weight_out)->x[i] ;\
}\
}\
\
void update_cloneuronesets_weight_in_base_##type(cloneuronset_##type * clnrnst){\
type sumDw=0;\
size_t nb_clone=clnrnst->nb_clone;\
@@ -659,45 +793,63 @@ void update_cloneuronesets_weight_in_base_##type(cloneuronset_##type * clnrnst){
sumDw=0;\
for(size_t c=0; c<nb_clone; ++c){\
sumDw += ((tmp_c[c])->weight_out)->x[i];\
\
\
}\
(tmp->weight_in)->x[i] += ((-1) * (tmp->learning_rate) * sumDw) / nb_clone ;\
(tmp->weight_in)->x[i] = (tmp->weight_in)->x[i] - ( (tmp->learning_rate) * sumDw) / nb_clone ;\
}\
for(size_t c=0; c<nb_clone; ++c){\
tmp_c[c]=(tmp_c[c])->next_layer;\
}\
tmp=tmp->next_layer;\
}\
/*while(tmp->next_layer){\
for(size_t c=0; c<nb_clone; ++c){\
tmp_c[c]=(tmp_c[c])->next_layer;\
}\
tmp=tmp->next_layer;\
}\
while(tmp != clnrnst->base){\
for(size_t i=0; i<((tmp->weight_in)->dim)->rank; ++i){\
sumDw=0;\
for(size_t c=0; c<nb_clone; ++c){\
(tmp_c[c])->TensorProduct(&((tmp_c[c])->weight_out), (tmp_c[c])->input, (tmp_c[c])->delta_out, (tmp_c[c])->nb_prod_thread);\
sumDw += ((tmp_c[c])->weight_out)->x[i];\
\
\
}\
(tmp->weight_in)->x[i] = (tmp->weight_in)->x[i] - ( (tmp->learning_rate) * sumDw) / nb_clone ;\
}\
for(size_t c=0; c<nb_clone; ++c){\
tmp_c[c]=(tmp_c[c])->prev_layer;\
}\
tmp=tmp->prev_layer;\
}*/\
free(tmp_c);\
}\
\
type clon_error_batch_##type(cloneuronset_##type * clnrnst){\
\
type err=0;\
type sumErrP=0;\
size_t nb_clone=clnrnst->nb_clone;\
neurons_##type **tmp_c=malloc(nb_clone*sizeof(neurons_##type *));\
for(size_t c=0; c<nb_clone; ++c)\
tmp_c[c] = (clnrnst->cloneurons[c])->next_layer;\
neurons_##type *tmp=(clnrnst->base)->next_layer;\
while(tmp){\
if(tmp->next_layer==NULL){\
for(size_t i=0; i<((tmp->target)->dim)->rank; ++i){\
sumErrP=0;\
for(size_t c=0; c<nb_clone; ++c){\
sumErrP += (tmp_c[c])->L(((tmp_c[c])->target)->x[i],((tmp_c[c])->output)->x[i]);\
}\
err += sumErrP/nb_clone;\
}\
break;\
}\
for(size_t c=0; c<nb_clone; ++c){\
tmp_c[c] = (clnrnst->cloneurons[c]);\
neurons_##type *tmp=(clnrnst->base);\
while(tmp->next_layer){\
for(size_t c=0; c<nb_clone; ++c)\
tmp_c[c]=(tmp_c[c])->next_layer;\
}\
tmp=tmp->next_layer;\
}\
err=0;\
for(size_t i=0; i<((tmp->target)->dim)->rank; ++i){\
for(size_t c=0; c<nb_clone; ++c){\
err += (tmp_c[c])->L(((tmp_c[c])->target)->x[i],((tmp_c[c])->output)->x[i]);\
}\
}\
free(tmp_c);\
\
return err / (((tmp->target)->dim)->rank);\
return err / (nb_clone * ((tmp->target)->dim)->rank);\
}\
\
struct arg_learnCloneuronset_##type{\
@@ -721,9 +873,10 @@ void* run_learnCloneuronset_thread_##type(void *arg){\
neurons_##type *tmp, *ttmp;\
while(!(*ending)){\
sem_wait(semaphore_datas);\
if(!(*ending)) break;\
init_copy_in_out_networks_from_tensors_##type(base_c, dataset->input[id_datas[id_th]],dataset->target[id_datas[id_th]]);\
tmp=base_c->next_layer;\
/*if((*ending)) break;\
*/init_copy_in_out_networks_from_tensors_##type(base_c, dataset->input[id_datas[id_th]],dataset->target[id_datas[id_th]]);\
/*printf(" id_datas [%ld] = %ld\n",id_th,id_datas[id_th]);\
*/tmp=base_c->next_layer;\
while(tmp){\
calc_out_neurons_##type(tmp);\
ttmp = tmp;\
@@ -731,9 +884,10 @@ void* run_learnCloneuronset_thread_##type(void *arg){\
}\
while(ttmp != base_c){\
calc_delta_neurons_##type(ttmp);\
/*update_weight_neurons_##type(ttmp);\
*/\
ttmp->TensorProduct(&(ttmp->weight_out), ttmp->input, ttmp->delta_out, ttmp->nb_prod_thread);\
/*only_update_weight_neurons_##type(ttmp);\
*//*update_weight_neurons_##type(ttmp);\
*/\
ttmp = ttmp->prev_layer;\
}\
sem_post(semaphore_learn);\
@@ -741,7 +895,7 @@ void* run_learnCloneuronset_thread_##type(void *arg){\
sem_post(semaphore_learn);\
}\
\
size_t learning_cloneuronset_##type(cloneuronset_##type *clnrnst, data_set_##type *dataset, neurons_##type *base, bool (*condition)(type, size_t)){\
size_t learning_cloneuronset_##type(cloneuronset_##type *clnrnst, data_set_##type *dataset, bool (*condition)(type, size_t)){\
size_t nbreps=0;\
size_t curData=0;\
type err=0;\
@@ -765,8 +919,10 @@ size_t learning_cloneuronset_##type(cloneuronset_##type *clnrnst, data_set_##typ
arg_th[i]->ending=ending;\
arg_th[i]->base_c = clnrnst->cloneurons[i] ;\
arg_th[i]->dataset = dataset ;\
arg_th[i]->id_datas= id_datas ;\
\
pthread_create(&thrd[i], NULL, run_learnCloneuronset_thread_##type, (void*)arg_th[i]);\
\
}\
\
\
@@ -785,9 +941,17 @@ size_t learning_cloneuronset_##type(cloneuronset_##type *clnrnst, data_set_##typ
sem_wait(semaphore_learn);\
}\
\
/*neurons_##type *tmp = (clnrnst->cloneurons[0])->next_layer;\
while(tmp){\
only_update_weight_neurons_##type(tmp);\
tmp=tmp->next_layer;\
}*/\
update_cloneuronesets_weight_in_base_##type(clnrnst);\
err = clon_error_batch_##type(clnrnst);\
*ending = condition(err, nbreps++) ;\
err = ABSMAX(err,clon_error_batch_##type(clnrnst));\
\
/*err = ABSMAX(err,error_out_##type(clnrnst->cloneurons[0]));\
*/nbreps += nb_clone;\
*ending = condition(err, nbreps) ;\
}\
\
printf("reps batch learning : %ld\n",nbreps);\
@@ -809,7 +973,7 @@ size_t learning_cloneuronset_##type(cloneuronset_##type *clnrnst, data_set_##typ
free(semaphore_learn);\
free(ending);\
free(id_datas);\
return err;\
return nbreps;\
} \
\
neuron_t/src/neuron_t/neuron_t.h
+16 -4
View File
@@ -8,6 +8,8 @@
//#include "tools_t/tools_t.h"
#include "tensor_t/tensor_t.h"
extern bool randomizeInitWeight;
struct config_layers{
size_t nb_layers;
size_t *sz_layers;
@@ -23,9 +25,14 @@ void free_config_layers(config_layers *pconf);
\
struct neurons_##type {/* layer */\
size_t id_layer;\
size_t iteration_step;\
size_t nb_prod_thread;\
size_t nb_calc_thread;\
type initial_learning_rate;\
type learning_rate;\
type decay_rate;\
size_t drop_rate;\
void (*update_learning_rate)(struct neurons_##type *N); \
tensor_##type *input; \
tensor_##type *net; /* output tensor_prodContract */\
tensor_##type *output; \
@@ -50,6 +57,10 @@ struct func_act_##type {\
type (*deriv_func_act)(type x); /* derivate func act */\
};\
\
void do_not_update_learnig_rate_##type(neurons_##type *N);\
void time_based_update_learning_rate_##type(neurons_##type *nr);\
void step_based_update_learning_rate_##type(neurons_##type *nr);\
void setup_learning_rate_params_neurons_##type(neurons_##type *base,type initial_learning_rate, type decay_rate, size_t drop_rate, void (*update_learning_rate)(neurons_##type *));\
/*void calc_net_neurons_##type(neurons_##type *nr);*/\
void calc_out_neurons_##type(neurons_##type *nr);\
void calc_delta_neurons_##type(neurons_##type *nr);\
@@ -58,11 +69,12 @@ void update_weight_neurons_##type(neurons_##type *nr);\
void init_copy_in_out_networks_from_tensors_##type(neurons_##type *nr, tensor_##type *input, tensor_##type *target);\
void init_in_out_networks_from_tensors_##type(neurons_##type *nr, tensor_##type *input, tensor_##type *target, neurons_##type *base);\
void init_in_out_all_networks_##type(neurons_##type *nr, tensor_##type *in, tensor_##type *out);\
void setup_networks_alloutputs_##type(neurons_##type **base_nr, size_t **array_dim_in_layers, size_t *sz_layers, size_t nb_layers);\
void setup_networks_alloutputs_config_##type(neurons_##type **base_nr, config_layers *lconf);\
void setup_networks_alloutputs_GLOBAL_rdm01_##type(neurons_##type **base_nr, size_t **array_dim_in_layers, size_t *sz_layers, size_t nb_layers);\
void setup_networks_alloutputs_##type(neurons_##type **base_nr, size_t **array_dim_in_layers, size_t *sz_layers, size_t nb_layers, bool randomize, type minR, type maxR, int randomRange);\
void setup_networks_alloutputs_config_##type(neurons_##type **base_nr, config_layers *lconf, bool randomize, type minR, type maxR, int randomRange);\
void setup_networks_layers_without_weights_##type(neurons_##type **base_nr, size_t **array_dim_in_layers, size_t *sz_layers, size_t nb_layers);\
void setup_networks_layers_without_weights_from_config_##type(neurons_##type **base, config_layers *pconf);\
void setup_networks_OneD_##type(neurons_##type **base_nr, size_t *array_dim_in_layers, size_t nb_layers);\
void setup_networks_OneD_##type(neurons_##type **base_nr, size_t *array_dim_in_layers, size_t nb_layers, bool randomize, type minR, type maxR, int randomRange);\
void init_in_out_all_networks_OneD_##type(neurons_##type *nr, type *in, size_t sz_in, type *out, size_t sz_out);\
void print_neurons_msg_##type(neurons_##type *nr, char * msg);\
\
@@ -106,7 +118,7 @@ struct cloneuronset_##type{\
typedef struct cloneuronset_##type cloneuronset_##type;\
void free_cloneuronset_##type(cloneuronset_##type *clnrnst);\
cloneuronset_##type * create_cloneuronset_from_base_conf_##type(neurons_##type *base, config_layers *conf, size_t nb_clone);\
size_t learning_cloneuronset_##type(cloneuronset_##type *clnrnst, data_set_##type *dataset, neurons_##type *base, bool (*condition)(type, size_t));\
size_t learning_cloneuronset_##type(cloneuronset_##type *clnrnst, data_set_##type *dataset, bool (*condition)(type, size_t));\
GEN_NEURON_(TYPE_FLOAT)
GEN_NEURON_(TYPE_DOUBLE)
neuron_t/test/is_good.c
+122 -11
View File
@@ -40,7 +40,7 @@ float df(float x){
TEST(init_One){
//endian=false;
neurons_TYPE_FLOAT *bn=NULL, *tmp=NULL, *ttmp=NULL;
setup_networks_OneD_TYPE_FLOAT(&bn, (size_t[]){3,5,2},3);
setup_networks_OneD_TYPE_FLOAT(&bn, (size_t[]){3,5,2},3,false,0,1,5000);
init_in_out_all_networks_OneD_TYPE_FLOAT(bn,(float[]){1.2,0.5,1.3},3,(float[]){0.1,0.8},2);
@@ -92,17 +92,18 @@ TEST(data_set_from_file){
#define epsilon 0.0001
bool cond(float e, size_t nbreps){
if (nbreps > 1) return true;
if (nbreps > 20000) return true;
if ((e<epsilon) && (e>-epsilon)) return true;
return false;
}
TEST(learning_first){
bool rec_randomizeInitWeight = randomizeInitWeight;
randomizeInitWeight =false;
data_set_TYPE_FLOAT *ds= fill_data_set_from_file_TYPE_FLOAT("xor.txt",1);
print_data_set_msg_TYPE_FLOAT(ds,"data");
neurons_TYPE_FLOAT *bn=NULL, *tmp ;
setup_networks_OneD_TYPE_FLOAT(&bn, (size_t[]){2,4,1},3); /* 2 input , 1 target; 1 hidden layer with 5 neurons */
setup_networks_OneD_TYPE_FLOAT(&bn, (size_t[]){2,4,1},3,false,0,1,5000); /* 2 input , 1 target; 1 hidden layer with 5 neurons */
setup_all_layers_functions_TYPE_FLOAT(bn,
tensorContractnProdThread_TYPE_FLOAT,
@@ -112,7 +113,7 @@ TEST(learning_first){
f,
df);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.5);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.1);
size_t reps = learning_online_neurons_TYPE_FLOAT(bn,ds,cond);
@@ -120,7 +121,8 @@ TEST(learning_first){
//char msg[256];
for(size_t i=0; i<ds->size; ++i){
print_predict_by_network_neurons_TYPE_FLOAT(bn,ds->input[i]);
print_predict_by_network_with_error_neurons_TYPE_FLOAT(bn,ds->input[i],ds->target[i]);
//print_predict_by_network_neurons_TYPE_FLOAT(bn,ds->input[i]);
/*sprintf(msg, "data set [%ld]",i);
init_copy_in_out_networks_from_tensors_TYPE_FLOAT(bn, ds->input[i],ds->target[i]);\
tmp=bn->next_layer;\
@@ -137,16 +139,19 @@ TEST(learning_first){
free_neurons_TYPE_FLOAT(bn);
LOG("reps = %ld\n",reps);
randomizeInitWeight = rec_randomizeInitWeight;
}
TEST(learning_second){
bool rec_randomizeInitWeight = randomizeInitWeight;
randomizeInitWeight =false;
data_set_TYPE_FLOAT *ds= fill_data_set_from_file_TYPE_FLOAT("xor.txt",1);
// print_data_set_msg_TYPE_FLOAT(ds,"data");
neurons_TYPE_FLOAT *bn=NULL, *tmp ;
setup_networks_OneD_TYPE_FLOAT(&bn, (size_t[]){2,4,1},3); /* 2 input , 1 target; 1 hidden layer with 5 neurons */
setup_networks_OneD_TYPE_FLOAT(&bn, (size_t[]){2,4,1},3,false,0,1,5000); /* 2 input , 1 target; 1 hidden layer with 5 neurons */
setup_all_layers_functions_TYPE_FLOAT(bn,
tensorContractnProdThread_TYPE_FLOAT,
@@ -156,7 +161,7 @@ TEST(learning_second){
f,
df);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.5);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 3 , 0.1);
size_t reps = learning_online2_neurons_TYPE_FLOAT(bn,ds,cond);
@@ -164,7 +169,8 @@ TEST(learning_second){
char msg[256];
for(size_t i=0; i<ds->size; ++i){
print_predict_by_network_neurons_TYPE_FLOAT(bn,ds->input[i]);
print_predict_by_network_with_error_neurons_TYPE_FLOAT(bn,ds->input[i],ds->target[i]);
//print_predict_by_network_neurons_TYPE_FLOAT(bn,ds->input[i]);
/*sprintf(msg, "data set [%ld]",i);
init_copy_in_out_networks_from_tensors_TYPE_FLOAT(bn, ds->input[i],ds->target[i]);\
tmp=bn->next_layer;\
@@ -182,15 +188,19 @@ TEST(learning_second){
free_neurons_TYPE_FLOAT(bn);
LOG("reps = %ld\n",reps);
randomizeInitWeight = rec_randomizeInitWeight;
}
TEST(learning_withconfig2){
bool rec_randomizeInitWeight = randomizeInitWeight;
randomizeInitWeight =false;
data_set_TYPE_FLOAT *ds= fill_data_set_from_file_TYPE_FLOAT("xor.txt",1);
// print_data_set_msg_TYPE_FLOAT(ds,"data");
config_layers *pconf = create_config_layers_from_OneD(3,(size_t[]){2,4,1}); /* 2 input , 1 target; 1 hidden layer with 5 neurons */
neurons_TYPE_FLOAT *bn=NULL, *tmp ;
setup_networks_alloutputs_config_TYPE_FLOAT(&bn,pconf);
//setup_networks_alloutputs_config_GLOBAL_rdm01_TYPE_FLOAT(setup_networks_alloutputs_config_TYPE_FLOAT(&bn,pconf);bn,pconf);
setup_networks_alloutputs_config_TYPE_FLOAT(&bn,pconf,false,0,1,5000);
setup_all_layers_functions_TYPE_FLOAT(bn,
tensorContractnProdThread_TYPE_FLOAT,
@@ -200,7 +210,7 @@ TEST(learning_withconfig2){
f,
df);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.5);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.1);
size_t reps = learning_online2_neurons_TYPE_FLOAT(bn,ds,cond);
@@ -218,9 +228,110 @@ TEST(learning_withconfig2){
free_neurons_TYPE_FLOAT(bn);
LOG("reps = %ld\n",reps);
randomizeInitWeight = rec_randomizeInitWeight;
}
TEST(learning_cloneuroneset){
bool rec_randomizeInitWeight = randomizeInitWeight;
randomizeInitWeight =false;
data_set_TYPE_FLOAT *ds= fill_data_set_from_file_TYPE_FLOAT("xor.txt",1);
// print_data_set_msg_TYPE_FLOAT(ds,"data");
config_layers *pconf = create_config_layers_from_OneD(3,(size_t[]){2,4,1}); /* 2 input , 1 target; 1 hidden layer with 5 neurons */
neurons_TYPE_FLOAT *bn=NULL, *tmp ;
//setup_networks_alloutputs_config_GLOBAL_rdm01_TYPE_FLOAT(setup_networks_alloutputs_config_TYPE_FLOAT(&bn,pconf);bn,pconf);
setup_networks_alloutputs_config_TYPE_FLOAT(&bn,pconf,false,0,1,5000);
setup_all_layers_functions_TYPE_FLOAT(bn,
tensorContractnProdThread_TYPE_FLOAT,
tensorProdThread_TYPE_FLOAT,
DL,
L,
f,
df);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.1);
//print_neurons_msg_TYPE_FLOAT(bn,"before create clones");
cloneuronset_TYPE_FLOAT *clnrnst = create_cloneuronset_from_base_conf_TYPE_FLOAT(bn, pconf, 3);
// size_t reps = learning_online2_neurons_TYPE_FLOAT(bn,ds,cond);
size_t reps = learning_cloneuronset_TYPE_FLOAT(clnrnst, ds,cond);
char msg[256];
for(size_t i=0; i<ds->size; ++i){
print_predict_by_network_with_error_neurons_TYPE_FLOAT(bn,ds->input[i],ds->target[i]);
}
free_cloneuronset_TYPE_FLOAT(clnrnst);
free_data_set_TYPE_FLOAT(ds);
free_neurons_TYPE_FLOAT(bn);
LOG("reps = %ld\n",reps);
randomizeInitWeight = rec_randomizeInitWeight;
}
TEST(learning_cloneuroneset_LEARN_RATE){
bool rec_randomizeInitWeight = randomizeInitWeight;
randomizeInitWeight =false;
data_set_TYPE_FLOAT *ds= fill_data_set_from_file_TYPE_FLOAT("xor.txt",1);
// print_data_set_msg_TYPE_FLOAT(ds,"data");
config_layers *pconf = create_config_layers_from_OneD(3,(size_t[]){2,4,1}); /* 2 input , 1 target; 1 hidden layer with 5 neurons */
neurons_TYPE_FLOAT *bn=NULL, *tmp ;
//setup_networks_alloutputs_config_GLOBAL_rdm01_TYPE_FLOAT(setup_networks_alloutputs_config_TYPE_FLOAT(&bn,pconf);bn,pconf);
setup_networks_alloutputs_config_TYPE_FLOAT(&bn,pconf,false,0,1,5000);
setup_all_layers_functions_TYPE_FLOAT(bn,
tensorContractnProdThread_TYPE_FLOAT,
tensorProdThread_TYPE_FLOAT,
DL,
L,
f,
df);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.4);
float initRate=0.6;
float decayRate=0.85; /* halving*/
size_t dropRate = 100;
// setup_learning_rate_params_neurons_TYPE_FLOAT(bn, initRate, decayRate, dropRate, time_based_update_learning_rate_TYPE_FLOAT);
setup_learning_rate_params_neurons_TYPE_FLOAT(bn, initRate, decayRate, dropRate, step_based_update_learning_rate_TYPE_FLOAT);
//print_neurons_msg_TYPE_FLOAT(bn,"before create clones");
cloneuronset_TYPE_FLOAT *clnrnst = create_cloneuronset_from_base_conf_TYPE_FLOAT(bn, pconf, 3);
// size_t reps = learning_online2_neurons_TYPE_FLOAT(bn,ds,cond);
size_t reps = learning_cloneuronset_TYPE_FLOAT(clnrnst, ds,cond);
char msg[256];
for(size_t i=0; i<ds->size; ++i){
print_predict_by_network_with_error_neurons_TYPE_FLOAT(bn,ds->input[i],ds->target[i]);
}
free_cloneuronset_TYPE_FLOAT(clnrnst);
free_data_set_TYPE_FLOAT(ds);
free_neurons_TYPE_FLOAT(bn);
LOG("reps = %ld\n",reps);
randomizeInitWeight = rec_randomizeInitWeight;
}
int main(int argc, char **argv){
neuron_t/test/xor.txt
+1 -1
View File
@@ -1,5 +1,5 @@
[*,2,1]
((1,0),1)
((0,0),0)
((1,1),0)
((1,0),1)
((0,0),0)