fanasina/y_project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug : nan, it's the learning rate too high,s try to find optimum: 0.001

Browse Source
This commit is contained in:
fanasina
2025-12-13 05:00:23 +01:00
parent 32207170f6
commit 3d54639d4c
18 changed files with 834 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
neuron_t/Makefile
+1 -1
View File
@@ -18,7 +18,7 @@ INCLUDE=-I$(NEURODIR)/src -I$(YPERMDIR)/src -I$(DIMDIR)/src -I$(TENSDIR)/src #"-
#LDFLAGS=-L$(YTESTDIR) -lytest -lOpenCL -lm -lpthread
#CFLAGS= -Wall -Werror -fpic $(INCLUDE)
CFLAGS= -Wall -Werror -fpic $(INCLUDE)
LDFLAGS= -lOpenCL -lpthread
LDFLAGS= -lOpenCL -lpthread
#SRC_DIR=$(ROOT_DIR)/src
#SRC=$(wildcard */*/*.c)
neuron_t/src/neuron_t/neuron_t.c
+30 -11
View File
@@ -57,7 +57,8 @@ type power_##type(type b, size_t p){\
void step_based_update_learning_rate_##type(neurons_##type *nr){\
nr->learning_rate=(nr->initial_learning_rate)*power_##type((nr->decay_rate),(1+(nr->iteration_step))/(nr->drop_rate));\
}\
\
type id_##type(type x){ return x;}\
type d_id_##type(type x){ return 1;}\
void setup_learning_rate_params_neurons_##type(neurons_##type *base,type initial_learning_rate, type decay_rate, size_t drop_rate, void (*update_learning_rate)(neurons_##type *)){\
while(base){\
base->initial_learning_rate = initial_learning_rate;\
@@ -123,10 +124,13 @@ void calc_delta_neurons_##type(neurons_##type *nr){\
/*decrement_dim_var(temp_w_d->dim);*/\
\
if(nr->nb_calc_thread < 2){\
for(size_t i = 0; i<(nr->net)->dim->rank; ++i)\
for(size_t i = 0; i<(nr->net)->dim->rank; ++i){\
if(temp_w_d->x[i]!=temp_w_d->x[i]) printf("debug: temp_w_d[%ld]=nan ",i);\
if((nr->net)->x[i] != (nr->net)->x[i]) printf("debug : (nr->net)->x[%ld] = nan ",i) ;\
(nr->delta_out)->x[i]=(nr->d_f_act)((nr->net)->x[i]) * temp_w_d->x[i] ;\
if((nr->delta_out)->x[i] != (nr->delta_out)->x[i] ) printf("debug: (nr->delta_out)->x[%ld]=nan ",i);\
/*print_tensor_msg_##type(nr->delta_out," nr delta_out calc 1 core hidden delta_out");\
*/\
*/}\
}else{\
update_4tensor_func_##type(nr->delta_out, nr->net, temp_w_d,\
funcalc_delta_hidden_out_##type , \
@@ -140,6 +144,8 @@ void calc_delta_neurons_##type(neurons_##type *nr){\
}\
\
type func_only_weight_in_##type(type w0, type w1, type scalar){\
if(w0 != w0) printf("debug: w0=nan ");\
if(w1 != w1) printf("debug: w1=nan ");\
return w0 - scalar * w1;\
}\
void only_update_weight_neurons_##type(neurons_##type *nr){\
@@ -703,6 +709,7 @@ void print_data_set_msg_##type(data_set_##type *ds, char *msg){\
size_t learning_online_neurons_##type(neurons_##type *base, data_set_##type *dataset, bool (*condition)(type,size_t)){\
neurons_##type *tmp=NULL, *ttmp;\
size_t nbreps=0;\
/*char strNbreps[128];*/\
do{\
for(size_t i=0; i<dataset->size; ++i){\
init_copy_in_out_networks_from_tensors_##type(base, dataset->input[i],dataset->target[i]);\
@@ -714,11 +721,18 @@ size_t learning_online_neurons_##type(neurons_##type *base, data_set_##type *dat
}\
while(ttmp != base){\
calc_delta_neurons_##type(ttmp);\
update_weight_neurons_##type(ttmp);\
/*update_weight_neurons_##type(ttmp);*/\
ttmp = ttmp->prev_layer;\
}\
ttmp = base->next_layer;\
while(ttmp){\
update_weight_neurons_##type(ttmp);\
ttmp = ttmp->next_layer;\
}\
}\
nbreps += (dataset->size);\
/*sprintf(strNbreps, " base %ld ",nbreps);\
print_neurons_msg_##type(base, strNbreps ); getchar();*/\
}while(!condition(error_out_##type(base), nbreps));\
\
\
@@ -730,8 +744,9 @@ size_t learning_online2_neurons_##type(neurons_##type *base, data_set_##type *da
size_t nbreps=0;\
type err=0;\
bool ending=false;\
/*char strNbreps[128];*/\
do{\
for(size_t i=0; i<dataset->size && !ending; ++i){\
for(size_t i=0; i<dataset->size /*&& !ending*/; ++i){\
init_copy_in_out_networks_from_tensors_##type(base, dataset->input[i],dataset->target[i]);\
tmp=base->next_layer;\
while(tmp){\
@@ -744,20 +759,23 @@ size_t learning_online2_neurons_##type(neurons_##type *base, data_set_##type *da
/*update_weight_neurons_##type(ttmp);\
*/ttmp = ttmp->prev_layer;\
}\
tmp = ttmp->next_layer;\
tmp = base/*ttmp*/->next_layer;\
while(tmp){\
update_weight_neurons_##type(tmp);\
tmp = tmp->next_layer;\
}\
err = ABSMAX(err,error_out_##type(base));\
/*if(i%20==0){err = error_out_##type(base);} else err = ABSMAX(err,error_out_##type(base));\
*/err = error_out_##type(base);\
ending = condition(err, ++nbreps);\
/*sprintf(strNbreps, " base %ld ",nbreps );\
print_neurons_msg_##type(base, strNbreps ); getchar();*/\
}\
\
}while(!ending);\
\
\
printf(" ### reps : %ld, err:%f \n",nbreps,err);\
return nbreps;\
/*printf(" ### reps : %ld, err:%f \n",nbreps,err);\
*/return nbreps;\
}\
\
neurons_##type * calculate_output_by_network_neurons_##type(neurons_##type *base, tensor_##type *input, tensor_##type **output_link){\
@@ -804,8 +822,8 @@ void print_predict_by_network_with_error_neurons_##type(neurons_##type *base, te
}\
\
\
printf(" error : %f\n", error_out_##type(base));\
print_tensor_msg_##type(input,"from input:");\
/*printf(" error : %f\n", error_out_##type(base));\
*/print_tensor_msg_##type(input,"from input:");\
\
}\
\
@@ -1021,3 +1039,4 @@ size_t learning_cloneuronset_##type(cloneuronset_##type *clnrnst, data_set_##typ
GEN_NEURONS_F_(TYPE_FLOAT)
GEN_NEURONS_F_(TYPE_DOUBLE)
GEN_NEURONS_F_(TYPE_L_DOUBLE)
neuron_t/src/neuron_t/neuron_t.h
+13 -7
View File
@@ -60,6 +60,8 @@ struct func_act_##type {\
void do_not_update_learnig_rate_##type(neurons_##type *N);\
void time_based_update_learning_rate_##type(neurons_##type *nr);\
void step_based_update_learning_rate_##type(neurons_##type *nr);\
type id_##type(type x);\
type d_id_##type(type x);\
void setup_learning_rate_params_neurons_##type(neurons_##type *base,type initial_learning_rate, type decay_rate, size_t drop_rate, void (*update_learning_rate)(neurons_##type *));\
/*void calc_net_neurons_##type(neurons_##type *nr);*/\
void calc_out_neurons_##type(neurons_##type *nr);\
@@ -125,6 +127,7 @@ size_t learning_cloneuronset_##type(cloneuronset_##type *clnrnst, data_set_##typ
GEN_NEURON_(TYPE_FLOAT)
GEN_NEURON_(TYPE_DOUBLE)
GEN_NEURON_(TYPE_L_DOUBLE)
#define UPDATE_ATTRIBUTE_NEURONE_IN_ALL_LAYERS(type, neuronVar, attribute, value) \
@@ -164,21 +167,24 @@ GEN_NEURON_(TYPE_DOUBLE)
free(vmsg);\
}while(0);\
#define BASH_PRINT_ATTRIBUTE_TENS_IN_ALL_LAYERS(type, bash_arg, neuronVar, attribute, msg)\
#define BASH_PRINT_ATTRIBUTE_TENS_IN_ALL_LAYERS(type, bash_arg, neuronVar, attribute, msg, putIndexTens)\
do{\
neurons_##type *tmpn = neuronVar;\
char *vmsg=malloc(strlen(msg)+70);\
size_t i=0;\
size_t lenVMG=0;\
/*size_t i=0;\
size_t lenVMG=0;*/\
size_t lenBSH_T=0;\
while(tmpn){\
lenVMG = sprintf(vmsg,"%s layer %ld",msg,i++);\
BASH_WRITE_IF_EXIST(bash_arg,vmsg,lenVMG);\
/*lenVMG = sprintf(vmsg,"%s layer %ld",msg,i++);\
BASH_WRITE_IF_EXIST(bash_arg,vmsg,lenVMG);*/\
if(tmpn->attribute){\
char *bashSTR=NULL;\
lenBSH_T=sprint_tensor_##type(&bashSTR, tmpn->attribute, true);\
BASH_WRITE_IF_EXIST(bash_arg,bashSTR,lenBSH_T);\
lenBSH_T=sprint_tensor_##type(&bashSTR, tmpn->attribute, putIndexTens);\
if(tmpn/*->next_layer==NULL*/){\
BASH_WRITE_IF_EXIST(bash_arg,bashSTR,lenBSH_T);\
}\
if(bashSTR) free(bashSTR);\
/*if(lenBSH_T==0) getchar();*/\
}\
tmpn = tmpn->next_layer;\
}\
neuron_t/test/is_good.c
+99 -16
View File
@@ -38,6 +38,12 @@ float df(float x){
return exp(-x)/ ((1+exp(-x)) * (1+exp(-x)));
}
float __id_(float x){
return x;
}
float d__id_(float x){
return 1;
}
TEST(init_One){
//endian=false;
@@ -94,7 +100,7 @@ TEST(data_set_from_file){
#define epsilon 0.0001
bool cond(float e, size_t nbreps){
if (nbreps > 20000) return true;
if (nbreps > 2000) return true;
if ((e<epsilon) && (e>-epsilon)) return true;
return false;
}
@@ -106,6 +112,7 @@ TEST(learning_first){
print_data_set_msg_TYPE_FLOAT(ds,"data");
neurons_TYPE_FLOAT *bn=NULL, *tmp ;
setup_networks_OneD_TYPE_FLOAT(&bn, (size_t[]){2,4,1},3,false,0,1,5000); /* 2 input , 1 target; 1 hidden layer with 5 neurons */
//setup_networks_OneD_TYPE_FLOAT(&bn, (size_t[]){2,4,1},3,true,0,1,5000); /* 2 input , 1 target; 1 hidden layer with 5 neurons */
setup_all_layers_functions_TYPE_FLOAT(bn,
tensorContractnProdThread_TYPE_FLOAT,
@@ -115,11 +122,23 @@ TEST(learning_first){
f,
df);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.1);
setup_all_layers_params_TYPE_FLOAT(bn, 5/*5*/, 3/*1*/ , 0.5);
neurons_TYPE_FLOAT *ttmp=bn;
while(ttmp){
if(ttmp->next_layer == NULL){
ttmp->f_act=__id_;
ttmp->d_f_act=d__id_;
}
ttmp=ttmp->next_layer;
}
LOG("%s","setup done\n");
print_neurons_msg_TYPE_FLOAT(bn,"bn before ");
size_t reps = learning_online_neurons_TYPE_FLOAT(bn,ds,cond);
print_neurons_msg_TYPE_FLOAT(bn,"bn after ");
//char msg[256];
for(size_t i=0; i<ds->size; ++i){
@@ -136,16 +155,16 @@ TEST(learning_first){
*/
}
LOG("reps = %ld error=%f\n",reps, error_out_TYPE_FLOAT(bn));
free_data_set_TYPE_FLOAT(ds);
free_neurons_TYPE_FLOAT(bn);
LOG("reps = %ld\n",reps);
randomizeInitWeight = rec_randomizeInitWeight;
}
TEST(learning_second_PRINT){
TEST(learning_second_PRINT){ endian=false;
bool rec_randomizeInitWeight = randomizeInitWeight;
randomizeInitWeight =false;
@@ -162,8 +181,15 @@ TEST(learning_second_PRINT){
f,
df);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 3 , 0.1);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.4);
neurons_TYPE_FLOAT *ttmp=bn;
while(ttmp){
if(ttmp->next_layer == NULL){
ttmp->f_act=__id_;
ttmp->d_f_act=d__id_;
}
ttmp=ttmp->next_layer;
}
size_t reps = learning_online2_neurons_TYPE_FLOAT(bn,ds,cond);
@@ -214,8 +240,16 @@ TEST(learning_withconfig2){
f,
df);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.1);
setup_all_layers_params_TYPE_FLOAT(bn, 1, 1 , 0.5);
neurons_TYPE_FLOAT *ttmp=bn;
while(ttmp){
if(ttmp->next_layer == NULL){
ttmp->f_act=__id_;
ttmp->d_f_act=d__id_;
}
ttmp=ttmp->next_layer;
}
size_t reps = learning_online2_neurons_TYPE_FLOAT(bn,ds,cond);
@@ -230,6 +264,8 @@ TEST(learning_withconfig2){
free_data_set_TYPE_FLOAT(ds);
free_neurons_TYPE_FLOAT(bn);
free_config_layers(pconf);
LOG("reps = %ld\n",reps);
randomizeInitWeight = rec_randomizeInitWeight;
@@ -255,8 +291,15 @@ TEST(learning_cloneuroneset){
f,
df);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.1);
setup_all_layers_params_TYPE_FLOAT(bn, 1, 1 , 0.5);
neurons_TYPE_FLOAT *ttmp=bn;
while(ttmp){
if(ttmp->next_layer == NULL){
ttmp->f_act=__id_;
ttmp->d_f_act=d__id_;
}
ttmp=ttmp->next_layer;
}
//print_neurons_msg_TYPE_FLOAT(bn,"before create clones");
cloneuronset_TYPE_FLOAT *clnrnst = create_cloneuronset_from_base_conf_TYPE_FLOAT(bn, pconf, 3);
@@ -308,7 +351,14 @@ TEST(learning_cloneuroneset_LEARN_RATE){
size_t dropRate = 100;
// setup_learning_rate_params_neurons_TYPE_FLOAT(bn, initRate, decayRate, dropRate, time_based_update_learning_rate_TYPE_FLOAT);
setup_learning_rate_params_neurons_TYPE_FLOAT(bn, initRate, decayRate, dropRate, step_based_update_learning_rate_TYPE_FLOAT);
neurons_TYPE_FLOAT *ttmp=bn;
while(ttmp){
if(ttmp->next_layer == NULL){
ttmp->f_act=__id_;
ttmp->d_f_act=d__id_;
}
ttmp=ttmp->next_layer;
}
//print_neurons_msg_TYPE_FLOAT(bn,"before create clones");
cloneuronset_TYPE_FLOAT *clnrnst = create_cloneuronset_from_base_conf_TYPE_FLOAT(bn, pconf, 3);
@@ -331,7 +381,6 @@ TEST(learning_cloneuroneset_LEARN_RATE){
LOG("reps = %ld\n",reps);
randomizeInitWeight = rec_randomizeInitWeight;
}
TEST(copy_weight_in_neurons){
@@ -356,7 +405,14 @@ TEST(copy_weight_in_neurons){
df);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.1);
neurons_TYPE_FLOAT *ttmp=bn;
while(ttmp){
if(ttmp->next_layer == NULL){
ttmp->f_act=__id_;
ttmp->d_f_act=d__id_;
}
ttmp=ttmp->next_layer;
}
size_t reps = learning_online2_neurons_TYPE_FLOAT(bn,ds,cond);
@@ -396,6 +452,7 @@ TEST(copy_weight_in_neurons){
LOG("reps = %ld\n",reps);
randomizeInitWeight = rec_randomizeInitWeight;
free_config_layers(pconf);
}
@@ -435,7 +492,14 @@ TEST(Extract_weight_in_neurons){
df);
setup_all_layers_params_TYPE_FLOAT(cpyn, 5, 1 , 0.1);
neurons_TYPE_FLOAT *ttmp=bn;
while(ttmp){
if(ttmp->next_layer == NULL){
ttmp->f_act=__id_;
ttmp->d_f_act=d__id_;
}
ttmp=ttmp->next_layer;
}
EXTRACT_FILE_TO_TENSOR_ATTRIBUTE_NNEURONS(TYPE_FLOAT, cpyn, weight_in, ".ff_bn_weight_in__toExtract.txt")
// copy_weight_in_neurons_TYPE_FLOAT(cpyn, bn);
EXPORT_TO_FILE_TENSOR_ATTRIBUTE_IN_NNEURONS(TYPE_FLOAT, bn, weight_in, ".ff_bn_weight_in__toExtract___exp.txt")
@@ -463,6 +527,8 @@ TEST(Extract_weight_in_neurons){
LOG("reps = %ld\n",reps);
randomizeInitWeight = rec_randomizeInitWeight;
free_config_layers(pconf);
}
@@ -492,7 +558,14 @@ TEST(Extract_EXPORT_weight_in_neurons){
df);
setup_all_layers_params_TYPE_FLOAT(bn, 5, 1 , 0.1);
neurons_TYPE_FLOAT *ttmp=bn;
while(ttmp){
if(ttmp->next_layer == NULL){
ttmp->f_act=__id_;
ttmp->d_f_act=d__id_;
}
ttmp=ttmp->next_layer;
}
size_t reps = 1;// learning_online2_neurons_TYPE_FLOAT(bn,ds,cond);
EXPORT_TO_FILE_TENSOR_ATTRIBUTE_IN_NNEURONS(TYPE_FLOAT, bn, weight_in, ".ff_bn_weight_in__toCMP__.txt")
@@ -534,6 +607,8 @@ TEST(Extract_EXPORT_weight_in_neurons){
LOG("reps = %ld\n",reps);
randomizeInitWeight = rec_randomizeInitWeight;
free_config_layers(pconf);
}
@@ -596,7 +671,14 @@ TEST(Extract_EXPORT_weight_in_neurons_double){
doubledf);
setup_all_layers_params_TYPE_DOUBLE(cpyn, 5, 1 , 0.1);
neurons_TYPE_DOUBLE *ttmp=bn;
while(ttmp){
if(ttmp->next_layer == NULL){
ttmp->f_act=id_TYPE_DOUBLE;
ttmp->d_f_act=d_id_TYPE_DOUBLE;
}
ttmp=ttmp->next_layer;
}
// EXTRACT_FILE_TO_TENSOR_ATTRIBUTE_NNEURONS(TYPE_DOUBLE, cpyn, weight_in, ".ff_bn_weight_in__toExtract.txt")
// copy_weight_in_neurons_TYPE_DOUBLE(cpyn, bn);
@@ -623,6 +705,7 @@ TEST(Extract_EXPORT_weight_in_neurons_double){
LOG("reps = %ld\n",reps);
randomizeInitWeight = rec_randomizeInitWeight;
free_config_layers(pconf);
}