import java.util.regex.*; import java.io.*; import java.util.*; import java.net.*; import javax.swing.tree.*; import javax.swing.*; public class FindURLsTree2 { private TreeNode buildURLTree(DefaultMutableTreeNode parent, int currDepth, int maxDepth, int maxPages) { if (currDepth>maxDepth) { return parent; } else { String url = (String) parent.getUserObject(); System.out.println("Level " + currDepth + " of " + maxDepth + " Parsing " + url); try { //read file into buffer BufferedReader in = new BufferedReader(new InputStreamReader(new URL(url).openStream() ) ); StringBuffer s = new StringBuffer(); String line; while ((line = in.readLine()) != null) { s.append(line + " "); } //check for redirects Pattern pRedir = Pattern.compile("META HTTP-EQUIV=\"*Refresh\"*.*URL=\"*(.*?)\"*[\\s,>]", Pattern.CASE_INSENSITIVE); Matcher mRedir = pRedir.matcher(s); if (mRedir.find()) { System.out.println("Redirecting to " + mRedir.group(1)); parent.setUserObject(mRedir.group(1)); buildURLTree(parent,currDepth,maxDepth,maxPages); } //parse for urls Pattern p = Pattern.compile("href.*?http://\"*(.*?)\"*[\\s,>]", Pattern.CASE_INSENSITIVE); Matcher m = p.matcher(s); int pageCtr=0; TreeSet sGood = new TreeSet(); TreeSet sTested = new TreeSet(); while (m.find() && pageCtr <= maxPages) { String surl = "http://"+(String)m.group(1); System.out.print(" Testing " + pageCtr + " " + surl); try { if (sTested.add(surl)) { //if we havent already tested this URL, do so. If it is a bad url, exception will follow BufferedReader junk = new BufferedReader(new InputStreamReader(new URL(surl).openStream() ) ); } //if we get here, it is a good URL. if (sGood.add(surl)) { System.out.println(" ... adding " + pageCtr + " of " + maxPages); //it has not yet been added, so add it DefaultMutableTreeNode t = new DefaultMutableTreeNode(surl); parent.insert(t,pageCtr++); //recurse System.out.println("... recursing "); buildURLTree(t,currDepth+1,maxDepth,maxPages); } } catch (Exception e) { System.out.println(" ... not adding"); } } } catch (IOException e) { e.printStackTrace(); System.err.println(e); } return parent; } } public static void main(String[] args) { FindURLsTree2 ft = new FindURLsTree2(); DefaultMutableTreeNode root = new DefaultMutableTreeNode(args[0]); int maxDepth=2; int maxPages=4; ft.buildURLTree(root,0,maxDepth,maxPages); System.out.println("Done building "); JFrame f = new JFrame(); JTree jTree = new JTree(root); f.getContentPane().add(jTree); f.setSize(300,300); f.setLocation(100,100); f.setVisible(true); f.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); } }