/*
 * Created on Mar 13, 2005 by Sergei Vasilevsky Part of CLCA project
 */
package edu.uiuc.webcoord;

import java.net.*;
import java.io.*;
import java.util.logging.*;


public class FetchHTML {
	static Logger logger = Logger.getLogger("logger.log");
	
	public static String getHTMLString(String strUrl) {
		String strHTML = "";
		
		try {
			URL mURL = new URL(strUrl);
			URLConnection uc = mURL.openConnection();
			BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream()));

			String inputLine;

			while ((inputLine = in.readLine()) != null)
				strHTML += inputLine + "\n";
			in.close();
		} catch (MalformedURLException e) {
			System.out.println("URL malformed - execution terminated");
			logger.severe("URL malformed");
			//e.printStackTrace();
		} catch (UnknownHostException ex) {
			System.out.println("Could not connect to host - execution terminated");
			logger.severe("Could not connect to host");
			//ex.printStackTrace();
		} catch (IOException ex1) {
			System.out.println("Severe error - execution terminated");
			logger.severe("IOException");
			ex1.printStackTrace();
		}
		return strHTML;
	}
	
	public static void displayUsageInfo() {
		System.out.println("Usage: ");
		System.out.println("	java FetchHTML [-U url | -F filename] [-I] [-O outputfilename]");
		System.out.println("");
		System.out.println("	ex: java FetchHTML -U http://www.uiuc.edu");
		System.out.println("	ex: java FetchHTML -F test.html -I");
	}
	
	public static void setupLogger() {
		Handler handler = null;
		
		try {
			handler = new FileHandler("fetchHTML.log",true);
		} catch (IOException e) {
			System.out.println("Using the console handler");
			handler = new ConsoleHandler();
		}
		
		logger.addHandler(handler);
		handler.setFormatter(new SimpleFormatter());
		logger.setUseParentHandlers(false); // Don't send output to this logger's parent as well - avoids having log info echoed to console
		logger.info("Logger initialized");
	}
	
	public static void main(String[] args) {
		setupLogger();
		
		if (args.length >= 2 && args.length <= 5) {
			//System.out.println(getHTMLString(args[0]));
			
			//Check for which type of input: file (-F) or URL (-U)
			//Check for coord conversion to integer flag (-I)
			//Check for output file flag (-O)
						
			String fetchedHTML = null;
			String inputFilename = "";
			String outputFilename = "";
			String outputValidationFile = "validatedHTML.txt";
			boolean fileCheck = true;
			boolean convertToIntFlag = false;
			boolean errorOptionFlag = false;
			boolean inputFlag = false;
			boolean outputFlag = false;
			
			for (int i = 0;i < args.length;i++){
				char flag = args[i].charAt(1);
				
				switch(flag) {
				case 'U' :
					if (inputFlag == false && (i+1) <= args.length){
						System.out.println("Fetching html from URL " + args[i+1] + "...");
						logger.info("Using URL " + args[i+1]);
						fetchedHTML = getHTMLString(args[i+1]);

						//write html into a fetchedHTML.txt file
						if (fetchedHTML != null) {
							try {

								logger.info("Creating File/Object output stream...");
								FileOutputStream fileOut = new FileOutputStream("fetchedHTML.txt");
								PrintWriter out = new PrintWriter(fileOut);
								logger.info("Writing HTML to fetchedHTML.txt...");
								out.write(fetchedHTML);
								logger.info("Closing all output streams...");
								out.close();
								fileOut.close();
								inputFilename = "fetchedHTML.txt";
							} catch (IOException e) {
								e.printStackTrace();
							}
						}
						inputFlag = true;
						i++;
					} else {
						errorOptionFlag = true;
					}
					
					break;
					
				case 'F' :
					if (inputFlag == false && (i+1) <= args.length){
						inputFilename = args[i+1];
						logger.info("Using file " + inputFilename);
						System.out.println("Fetching html from file " + inputFilename + "...");
						File inputFile = new File(inputFilename);
						if ( !inputFile.exists() ) {
							fileCheck = false;
						}
						inputFlag = true;
						i++;
					} else {
						errorOptionFlag = true;
					}
					
					break;
					
				case 'I' :
					if (convertToIntFlag == false){
						System.out.println("Using coordinate conversion...");
						logger.info("Using coordinate conversion");
						convertToIntFlag = true;
					} else {
						errorOptionFlag = true;
					}
					
					break;
					
				case 'O' :
					if (outputFlag == false && (i+1) <= args.length){
						outputFilename = args[i+1];
						logger.info("Using output file " + outputFilename);
						System.out.println("Using output file " + outputFilename);
						outputFlag = true;
						i++;
					} else {
						errorOptionFlag = true;
					}
					
					break;
					
				default :
					errorOptionFlag = true;
				}
			}
			
			if (errorOptionFlag == false && inputFlag == true) {
				if ( fileCheck )  {
					System.out.println("Validating...");
					logger.info("Validating " + inputFilename);
					HTMLValidate hv = new HTMLValidate(inputFilename, outputValidationFile, "HTMLValidateError.log", false);
		        	hv.run();
		        	logger.info("Validation complete");
		        	
		        	boolean validateError = false;
		        	
		        	//check to make sure that the validator handled the file successfully
		    		try {
		    			FileReader fr = new FileReader("HTMLValidateError.log");
		    			BufferedReader in = new BufferedReader(fr);
		    			String inputLine;
		    		
		    		    //loop through validator log file
		    			while ((inputLine = in.readLine()) != null){
		    				if ( inputLine.indexOf("This document has errors that must be fixed") >= 0)
		    					validateError = true;
		    			}
		    		} catch (FileNotFoundException e) {
		    			System.out.println("Validation log file not found");
		    			logger.severe("Validation log file not found: HTMLValidateError.log");
		    		} catch (IOException ex) {
		    			ex.printStackTrace();
		    		}
		
		        
					System.out.println("Parsing...");
					logger.info("Parsing HTML from " + outputValidationFile);
				
					//Get HTML String from file
					fetchedHTML = getHTMLStringFile(outputValidationFile);
				
					if ( (fetchedHTML.length()> 0) && (!validateError) ) {
						
						HTMLDocument doc = HTMLParser.parseHTML(fetchedHTML);
						
						if (doc != null){
							if (convertToIntFlag == true){
								doc = doc.toIntCoords();
								if (doc==null) {
									System.out.println("Encountered Severe Error while converting to Integer Coordinates! Exiting...");
									System.exit(1);
								}
							}
				
							System.out.println("Creating output...");
						
							//write resulting object to outfile
							try {

								logger.info("Creating File/Object output stream...");
							
								FileOutputStream fileOut = null;
								PrintWriter out = null;
							
								if (outputFlag == true) {
									fileOut = new FileOutputStream(outputFilename);
									out = new PrintWriter(fileOut);
								} else {
									out = new PrintWriter(System.out);
								}
							
								logger.info("Writing Parsed HTML Object...");
								out.write(doc.toXMLString());

								logger.info("Closing all output streams...");
								out.close();
							
								if (outputFlag == true) {
									fileOut.close();
								}
							
								logger.info("Done!");

							} catch (FileNotFoundException e) {
								e.printStackTrace();
							} catch (IOException e) {
								e.printStackTrace();
							}
						}
					} else {
						
						if ( validateError) {
							System.out.println("The validator was unable to process this file or url.");
							logger.info("Validator unable to process file.");
						} else if ( (fetchedHTML.length()<= 0) ) {
							System.out.println("Empty validation output file - nothing to parse");
							logger.info("Empty validation output file.");
						}
					}
				
				} else {
					System.out.println("The input file " + inputFilename + " does not exist");
					logger.info("The input file " + inputFilename + " does not exist");
				}
				
			} else {
				logger.info("Incorrect option flags");
				displayUsageInfo();
			}
			
		} else {
			logger.info("Wrong number of arguments");
			displayUsageInfo();
		}
	}
	
	/**
	 * @param strFilePath
	 * @return
	 */
	public static String getHTMLStringFile(String strFilePath) {
		String fetchedHTML;
		String inputLine,tempString;
		fetchedHTML = "";
		tempString="";
		
		try {
			FileReader fr = new FileReader(strFilePath);
			BufferedReader in = new BufferedReader(fr);
		
			while ((inputLine = in.readLine()) != null){
				inputLine = tempString + inputLine;
			
				tempString = cleanString(inputLine);
				
				if (tempString.endsWith("<b>")){
					// '<b> at end of string signifies that temporary string has a fragment of a tag.
					// We will not reset temporary string and simply append next read line to it on
					// following loop iteration.  Need to remove '<b>' first!
					tempString = tempString.replace("<b>","");
				} else {
					fetchedHTML += tempString + "\n"; // Preserve newline as removing results in unexpected parsing behavior
					tempString = ""; // Reset temporary string as no tag fragments
				}
			}
			
			in.close();
			
			if (tempString.length() > 0) {
				// Should not happen as this indicates that we have read in all of file and we have
				// a tag fragment remaining.  Will continue processing anyway.  Log this event, however.
				logger.severe("Reached end of file with tag fragment!");
				System.out.println("Tag fragment encountered - attempt to parse anyway...");
				fetchedHTML += tempString + "\n";
			}
			
		} catch (FileNotFoundException e) {
			System.out.println("Validation output file not found - execution terminated");
			logger.severe("Validation output file not found: " + strFilePath);
		} catch (IOException ex) {
			ex.printStackTrace();
		}
		return fetchedHTML;
	}

	/**
	 * 
	 * @param inputString
	 * TODO Add support for additional tags and generalize for those that have optional attributes
	 * 
	 * The following tags are currently being handled by this method:
	 * 
	 * <abbr> </abbr>		: Remove start/end tags to define abbreviations (Strict DTD)
	 * <acronym> </acronym>	: Remove start/end tags to define acronyms (Strict DTD)
	 * <b> </b> 			: Remove start/end tags for bold text (Strict DTD)
	 * <basefont>			: Remove tag that sets the base font size (Transitional DTD. Deprecated in favor of style sheets.)
	 * <bdo> </bdo>			: Remove start/end tags for bi directional override (Strict DTD)
	 * <big> </big>			: Remove start/end tags for large text (Strict DTD)
	 * <cite> </cite>		: Remove start/end tags for reference citation (Strict DTD)
	 * <code> </code>		: Remove start/end tags for identifying a code fragment (Strict DTD)
	 * <del> </del>			: Remove start/end tags for showing text having been deleted (Strict DTD)
	 * <dfn> </dfn>			: Remove start/end tags for defining an enclosed term (Strict DTD)
	 * <em> </em>			: Remove start/end tags to show emphasis (Strict DTD)
	 * <font> </font>		: Remove start/end tags to change font size and color (Transitional DTD. Deprecated in favor of style sheets.)
	 * <i> </i> 			: Remove start/end tags for italicized text (Strict DTD)
	 * <ins> </ins>			: Remove start/end tags for showing text having been inserted (Strict DTD)
	 * <kbd> </kdb>			: Remove start/end tags to show text user would type (Strict DTD)
	 * <q> </q>				: Remove start/end tags to display short quotations (Strict DTD)
	 * <s> </s> 			: Remove start/end tags for strikethrough text (Transitional DTD. Deprecated.)
	 * <samp> </samp>		: Remove start/end tags for sample output (Strict DTD)
	 * <small> </small>		: Remove start/end tags for small text (Strict DTD)
	 * <span> </span>		: Remove start/end tags for generic inline container (Strict DTD)
	 * <strike> </strike> 	: Remove start/end tags for strikethrough text (Strict DTD)
	 * <strong> </strong>	: Remove start/end tags for stronger emphasis (Strict DTD)
	 * <sub> </sub>			: Remove start/end tags to create subscript (Strict DTD)
	 * <sup>				: Replace start tag to create superscript with '^' (Strict DTD)
	 * </sup>				: Remove end tag to create superscript (Strict DTD)
	 * <tt> </tt>			: Teletype (monospaced) text (Strict DTD)
	 * <u> </u> 			: Start/End tags for underlined text (Strict DTD)
	 * <var> </var>			: Start/End tags to show variable (Strict DTD)
	 *  
	 * @return
	 */
	public static String cleanString(String inputString) {
		String outputString = null;
		
		if (inputString != null) {
			// First replace complete tag occurences.  We will then need to check to see
			// whether some tags span across 2 or more lines (caused by the use of optional
			// attributes).
			
			outputString = inputString.replaceAll("<[bB] [^>]*>|<[bB]>|</[bB]>|<[iI] [^>]*>|<[iI]>|</[iI]>|<[uU] [^>]*>|<[uU]>|</[uU]>|<[sS] [^>]*>|<[sS]>|</[sS]>|</?[sS][mM][aA][lL]{2}[^>]*>","");
			outputString = outputString.replaceAll("</?[sS][tT][rR][iI][kK][eE][^>]*>|</?[tT]{2}[^>]*>|</?[bB][iI][gG][^>]*>|</?[sS][tT][rR][oO][nN][gG][^>]*>|</?[aA][cC][rR][oO][nN][yY][mM][^>]*>","");
			outputString = outputString.replaceAll("<[bB][aA][sS][eE][fF][oO][nN][tT][^>]*>|</?[fF][oO][nN][tT][^>]*>|</?[eE][mM][^>]*>|</?[kK][bB][dD][^>]*>|</?[aA][bB]{2}[rR][^>]*>|</?[cC][oO][dD][eE][^>]*>","");
			outputString = outputString.replaceAll("</?[vV][aA][rR][^>]*>|</?[sS][uU][bB][^>]*>|</[sS][uU][pP]>|</?[sS][pP][aA][nN][^>]*>|</?[sS][aA][mM][pP][^>]*>|</?[qQ][^>]*>|</?[cC][iI][tT][eE][^>]*>|</?[dD][fF][nN][^>]*>","");
			outputString = outputString.replaceAll("</?[dD][eE][lL][^>]*>|</?[iI][nN][sS][^>]*>|</?[bB][dD][oO][^>]*>","");
			outputString = outputString.replaceAll("<[sS][uU][pP][^>]*>","^");
			outputString = outputString.replaceAll("&nbsp;"," ");
			
			// Now check if new string has fragments of tags.  If it does then it means that tag spans
			// multiple lines.  We will add '<b>' at the end of the string to serve as a flag to the calling
			// method so that it can append another line to the string and call this method again.  The use
			// of '<b>' is deliberate as the method should have removed this tag.
			if((outputString.toUpperCase()).matches(".*<B .*|.*<Q .*|.*<I .*|.*<ABBR .*|.*<DEL .*|.*<INS .*|.*<SUB .*|.*<BDO .*|.*<SUP .*|.*<S .*|.*<U .*|.*<SMALL.*|.*<SAMP .*|.*<SPAN .*|.*<VAR .*|.*<ACRONYM .*|.*<CODE .*|.*<STRIKE.*|.*<TT .*|.*<DFN .*|.*<CITE .*|.*<KBD .*|.*<EM .*|.*<STRONG.*|.*<BASEFONT.*|.*<FONT.*|.*<BIG.*")) {
				outputString = outputString + "<b>"; // Fragment so tag spans multiple lines. Append flag to notify calling method.
			}
				
		}
		
		return outputString;
	}

}
