'''
Created on Mar 11, 2014

@author: Brett Paufler

Copyright Brett Paufler 2014
'''

print "Start website_ripper"

import re
import urllib2
import os

sourceFileLocation = "web_page_with_links.html"
sourceFile = open(sourceFileLocation, 'r')



def makeOutputDirectory(a,letter=0):
    print "makeOutputDirectory called"
    if letter:
        c = a + ".\\" + letter + ".\\"
    else:
        c = a
    if not os.path.exists(c):
        os.makedirs(c)
        print "Directory " + c + " was created"
    else:
        print "Directory " + c + " already existed"

new_dir = "new_dir"
makeOutputDirectory(new_dir)


# This is project specific, inputed by hand from html
aNameStart = '"0" alt=""> '
aNameEnd = '</a>'
aLinkStart = 'href="'
aLinkEnd = '/"><img'

successfulPages = 0
numberOfSomething = 0
as = []

for line in sourceFile:
    #print line
    if aNameStart in line:
        numberOfSomething += 1
        

        aName = re.search(re.escape(aNameStart)+"(.*?)"+re.escape(aNameEnd),line).group(1)
        aLink = re.search(re.escape(aLinkStart)+"(.*?)"+re.escape(aLinkEnd),line).group(1)
        print line
        print aName
        print aLink
        tempa = {}
        tempa["name"] = aName
        tempa["aLink"] = aLink
        as.append(tempa)


#for item in as:
#for item in as:


def nameFormat (thisa):
    name = thisa["name"]
    name = name.replace(" ", "")
    name = name.replace("@", "")
    name = name.replace("/", "")
    name = name.replace(".", "")
    name = ".\\asPages\\" + name + ".html"
    print name   
    return name


def downloadWebpage(thisURL, nameFile):
    global count
    global successfulPages
    print "DownLaoding " + thisURL
    
    if "@" in nameFile:
        nameFile = ".\\asPages\\" + count + '.html'
        
    if "/" in nameFile:
        nameFile = ".\\asPages\\" + count + '.html'
    
    if ".aol.com" in nameFile:
        nameFile = ".\\asPages\\" + count + '.html'     
             
    LocalWegPage = open(nameFile, 'wb')


    
    if thisURL.startswith("obfuscated_web_address"):
        if not "@" in thisURL:   
            u = urllib2.urlopen(thisURL)  
      
            LocalWegPage.write(u.read())
            LocalWegPage.close()
            successfulPages += 1
            print "DownLoad Finished"



# all cases
print "FOR ALL CASES"
for item in as:
    theURL = item["aLink"]

    saveAs = nameFormat(item)
    print "Working on This File in ALL CASES"    
    print theURL
    downloadWebpage(theURL, saveAs)





'''
# singleCase to Start  COMMENT OUT   
theURL = as[0]["aLink"]
saveAs = nameFormat(item)
print "Working on This SINGLE File"    
print theURL
print saveAs  
downloadWebpage(theURL, saveAs)
'''
 

print "There were " + str(numberOfSomething) + " as (should be 457)"
print "Length of as[] = " + str(len(as))
print "End pmGetData"