Hello All,
I am getting java.lang.OutOfMemoryError: GC overhead limit exceeded error when reading from a text file.I am not sure what is going wrong.I am running my program on a cluster having sufficient memory.The outer loop iterates for 16000 times and for each iteration of the outer loop the inner loop iterates for about 300,000 times.The error is thrown at different points in the program but it occurs only when I insert the code which stores the <String,Float> pair in a hashmap.Any suggestions will be grately appreciated.
while((line=br.readLine())!=null)
{
testCnt++;
if(testCnt==1)
continue;
leastFive.clear();
fiveTrainURLs.clear();
StringTokenizer st=new StringTokenizer(line," ");
while(st.hasMoreTokens())
{
String currentToken=st.nextToken();
if(currentToken.contains("File"))
{
testDataFileNo=st.nextToken();
String tok="";
while(st.hasMoreTokens())
{
tok=st.nextToken();
int topic_no=Integer.parseInt(tok);
String prob=st.nextToken();
double double_prob=Double.parseDouble(prob);
p1[topic_no]=double_prob;
}
break;
}
}
FileReader fr1=new FileReader("/homes/output_train_2000.txt");
BufferedReader br1=new BufferedReader(fr1);
String line1="";
int cnt=0;
String currentToken="";
int topic_no=0;
String prob="";
double double_prob=0;
String tok="";
while((line1=br1.readLine())!=null)
{
cnt++;
if(cnt==1)
continue;
for(int i=0;i<2001;i++)
p2[i]=0.0;
boolean flag=false;
StringTokenizer st1=new StringTokenizer(line1," ");
while(st1.hasMoreTokens())
{
currentToken=st1.nextToken();
if(flag==true)
{
topic_no=Integer.parseInt(currentToken);
prob=st1.nextToken();
//This is where the error is thrown
double_prob=Double.parseDouble(prob);
p2[topic_no]=double_prob;
}
if(currentToken.contains("File"))
{
trainDataFileNo=st1.nextToken();
flag=true;
}
}
flag=false;
double result=klDivergence(p1,p2);
float resultFloat=(float)result;
//The error is thrown if I introduce this line in the code
leastFive.put(trainDataFileNo,resultFloat);
line1="";
}
br1.close();
}
bw.close();
br.close();