import requests
from bs4 import BeautifulSoup
import pandas as pd

URL = "https://www.indeed.com/salaries/Data-Scientist-Salaries"

res = requests.get(URL)
page_source = BeautifulSoup(res.content,"lxml")

USA_respondents = int(page_source.find('div', class_="cmp-salary-header-content").text.split()[3].replace(",",""))
# this is the number of salary survey respondents
# which I will use as an indicator of size of industry in the location
print("salary info collected from {:,d} entities in the USA").format(USA_respondents)

state_list = page_source.findAll('option', {'data-tn-element':"loc_state[]"})
state_URLs = [i.attrs['value'] for i in state_list]
state_URLs[:3]

city_list = page_source.findAll('option', {'data-tn-element':"loc_city[]"})
city_URLs = [i.attrs['value'] for i in city_list]
city_URLs[:13]  # to show several states

USA_min_salary = page_source.find('div', class_="cmp-sal-min").text
print("{} is the minimum reported salary in the USA").format(USA_min_salary)

USA_max_salary = page_source.find('div', class_="cmp-sal-max").text
print("{} is the maximum reported salary in the USA").format(USA_max_salary)

USA_avg_salary = page_source.find('div', class_="cmp-sal-salary").text.encode('utf-8')
print("{} is the average reported salary in the USA").format(USA_avg_salary)

def extract_min_salary(location_source):
    a = location_source.find('div', class_="cmp-sal-min")
    return a.text.strip() if a else None

def extract_max_salary(location_source):
    a = location_source.find('div', class_="cmp-sal-max")
    return a.text.strip() if a else None

def extract_avg_salary(location_source):
    a = location_source.find('div', class_="cmp-sal-salary")
    return a.text.strip() if a else None

def extract_respondents(location_source):
    a = location_source.find('div', class_="cmp-salary-header-content")
    return a.text.split()[3] if a else None

state_results = []
for loc in state_URLs:
    state_res = requests.get("https://www.indeed.com"+loc)
    state_source = BeautifulSoup(state_res.content,"lxml")
    state_results.append(
                        (" ".join(loc.split("-")[3:]), \
                        extract_respondents(state_source), \
                        extract_min_salary(state_source), \
                        extract_max_salary(state_source), \
                        extract_avg_salary(state_source))
                        )

state_results

city_results = []
for loc in city_URLs:
    city_res = requests.get("https://www.indeed.com"+loc)
    city_source = BeautifulSoup(city_res.content,"lxml")
    city_results.append(
                        (" ".join(loc.split("-")[3:-1]),loc.split("-")[-1], \
                        extract_respondents(city_source), \
                        extract_min_salary(city_source), \
                        extract_max_salary(city_source), \
                        extract_avg_salary(city_source))
                        )

city_results

state_df = pd.DataFrame(state_results, columns=('state','number_resps',"min_salary","max_salary",'avg_salary'))
city_df = pd.DataFrame(city_results, columns=('city','State','number_resps',"min_salary","max_salary",'avg_salary'))
print state_df.head(2)
print
print city_df.head(2)

city_df.head(5)

def fix_salaries(salary_string):
    """
    takes salary string from indeed listing 
    and converts it to annual equivalient.
    """
    salary = 0
    salary_list = salary_string.replace("$","").replace(",","").strip().split()
    if "-" in salary_list[0]:
        temp = salary_list[0].split("-")
        salary = sum([float(a) for a in temp])/len(temp)
    else:
        salary = float(salary_list[0])
    if salary_list[-1] == "month":
        salary *= 12
    elif salary_list[-1] == "hour":
        salary *= 2000
    return salary
    

city_df['avg_salary'] = city_df['avg_salary'].map(fix_salaries)
state_df['avg_salary'] = state_df['avg_salary'].map(fix_salaries)

cities_money = city_df.sort_values(by='avg_salary',ascending=False).head(50)
cities_money.head(10)

city_df['number_resps'] = city_df['number_resps'].map(lambda x: x.replace(",","")).map(int)
state_df['number_resps'] = state_df['number_resps'].map(lambda x: x.replace(",","")).map(int)

cities_jobs = city_df.sort_values(by='number_resps',ascending=False).head(50)
cities_jobs.head(10)

cities_to_search = pd.concat([cities_jobs,cities_money])
cities_to_search.drop_duplicates(inplace=True)
[x[0].replace(" ","+") + "%2C+" + x[1] for x in list(cities_to_search[['city','State']].values)]

state_df.to_csv("states.csv")
city_df.to_csv("cities.csv")