If this is your first visit, be sure to
check out the FAQ by clicking the
link above. You may have to register
before you can post: click the register link above to proceed. To start viewing messages,
select the forum that you want to visit from the selection below.

// if no other page exists break out of loop
if (strType == null)
break;
if (strType.compareTo("text/html") == 0)
{
// if this URl is not searched already and not on the list to currentl search then do following..
if ((!vectorSearched.contains(strLink)) && (!vectorToSearch.contains(strLink)))
{

// test to make sure it is robot-safe!
if (robotSafe(urlLink))
// now add it to the list of ones we want to search
vectorToSearch.addElement(strLink);
}
}

// if the proper type, add it to the results list
// unless we have already seen it
if (strType.compareTo("text/html") == 0)
{
//vectorMatches.addElement(strLink);
for (int i =0; i < vectorToSearch.size(); i++)
{
System.out.println(i);
}

numberFound++;
if (numberFound >= SEARCH_LIMIT)
break;
// this is where we want to display the list orf URLs found to user

// read in entire file
byte b[] = new byte[1000];
// read in the stream of bytes and store it in an integer numRead
int numRead = urlRobotStream.read(b);
// now convert this file from the url into the string strCommands
strCommands = new String(b, 0, numRead);
// while numRead positive do the following checks
while (numRead != -1) {
// if currentThread different than the serachThread then break out of the loop and (continue on again???)
if (Thread.currentThread() != searchThread)
break;
numRead = urlRobotStream.read(b);
if (numRead != -1) {
String newCommands = new String(b, 0, numRead);
// new string now
strCommands += newCommands;
}
}
//close the stream
urlRobotStream.close();
} catch (IOException e) {
// if there is no robots.txt file, it is OK to search
// ok to search so return true to the method
return true;
}