#!/usr/bin/env python3
#coding: utf8

#Scraper
from bs4 import BeautifulSoup
#Connexion web
import mechanicalsoup
import wget
from os import mkdir, listdir
from re import findall
from time import sleep
from threading import Thread

'''
############## 4chan thread scrapper ################
This script is deisgned to look for specific words on 4 chan boards, and download all images from relevant thread.
Usage of a VPN is recommended since 4chan is a shady place. Use at your own risk !


   .  \\
   ,` ( `   SquiP
  (  \' "
   `-.__)_ 

'''




def getArgs():
    '''Gets all the arguments passed to the script and returns them in a parse_args()-type object.
    No args
    Returns:
    -args : an args object containing all the optional arguments passed to the script.
    '''
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("-u", "--url", help = "URL of the board. MUST INCLUDE FINAL /, ex : https://boards.4chan.org/b/", action="store", type=str, required=True)
    parser.add_argument("-f", "--folder", help = "Folder in which downloads will go, ex: ./downloads", action="store", type=str, required=True)
    parser.add_argument("-k", "--keyword", help = "keyword or phrase to look for in the threads, ex : 'cute thread'", action="store", type=str, required=True)
    parser.add_argument("-c", "--constant", help = "Constantly download", action="store_true")

    #Creating the args object
    args=parser.parse_args()

    return args

def thread_finder(homepage, keyword):
    '''
    returns a list of all the threads where op said keyword on 4chan homepage
    Args:
    - homepage: bs4 soup object containing html from the homepage of the board
    - keyword : any single word
    Returns:
    - hrefs : all the references to matching threads. They must be appended to homepage to work
    '''
    href = homepage.findAll('a', {'title': 'Reply to this post'})
    threads = homepage.findAll('div', {'class': 'thread'})
    hrefs = []

    for thread in threads:
        texts = thread.findAll('blockquote', {'class' : 'postMessage'})
        for text in texts:
            if keyword.lower() in text.text.lower() and "loli" not in text.text.lower() and "shota" not in text.text.lower():
                links = thread.findAll('a', {'title': 'Reply to this post'})
                for link in links:
                    hrefs.append(f"{link['href']}")
    return hrefs
    



def html_get(url):
    '''
    Get html from the webpage
    Args:
    - url : a str containing url to scrap
    Returns:
    - page.soup: A BeautifulSoup object containing html
    '''
    #Browser
    browser = mechanicalsoup.Browser()
    try:
        page = browser.get(url)
        print(f"Got html from {url}")
        return page.soup
    except Exception as e:
        print(f"Got error {e}")


def scraper(soup):
    '''
    Scrape a bs4 html object, find posts w/ images and get full size source
    Args:
    - soup : bs4 soup item
    - item : tag to find
    Returns:
    - sources : A list of image sources
    '''
    tags=soup.findAll('a', {"class": "fileThumb"})
    sources = []
    for tag in tags:
        sources.append(tag['href'])

    #Got image sources, removing any left http headers
    for index, item in enumerate(sources):
        sources[index] = item.replace("http://", "")
        sources[index] = item.replace("https://", "")
        sources[index] = item.replace("//", "")
    return sources


def item_dl(sources, dlfolder):
    '''
    Download all items in the sources list to folder dlfolder, which we try to create"
    Args:
    - sources : a list of URLsi
    - global folder_content : see folder_watch()

    '''
    
    global folder_content
    
    try:
    #Making folder
        mkdir(dlfolder)
    except FileExistsError:
        print(f"{dlfolder} already exists, not creating")

    #Deduplicating
    imagenames = []

    for source in sources:
        fullsource = "http://" + source
        imagename = findall(r"[^\/]*$", source)[0]
        if imagename[:-4] not in folder_content:
            name = wget.download(fullsource, out=dlfolder)
            print(f"{name} downloaded")

    return True

def folder_watch(folder):
    '''
    Watch for the content of a folder and return its content.
    Content is a string containing all the names of all the elements.
    Args:
    - folder : folder to watch
    - global folder_content : see folder_watch()
    Returns:
    folder_content : said string, containing all the names of all the files in the folder
    '''

    global folder_content

    folder_list = listdir(folder)
    folder_content = "" 

    for i in folder_list:
        folder_content += i


def dl_threads(folder, url):
    '''
    Constantly download...
    Args: 
    - folder: folder to dl into
    - url : board to watch
    '''

    try:
        sleep(2)
        soup = html_get(url)
        hrefs = thread_finder(soup, keyword)
        sources = scraper(soup)
        #item_dl(sources, folder)

        #Dling all threads found

        #oneshot
        for href in hrefs:
            print(f"going after {url}{href}")
            subsoup = html_get(f"{url}{href}")
            subsources = scraper(subsoup)
            folder_watch(folder)
            item_dl(subsources, folder)
    except Exception as e:
        print(f"Houston, we had a problem: \n{e}")



#Getting main elements
args = getArgs()
folder = args.folder
keyword = args.keyword

url = args.url
soup = html_get(url)
hrefs = thread_finder(soup, keyword)
sources = scraper(soup)
folder_content = ""
#item_dl(sources, folder)

#Dling all threads found

#oneshot
if not args.constant:
    for href in hrefs:
        folder_watch(folder)
        dl_threads(folder_url)
else:
    while True:
            folder_watch(folder)
            dl_threads(folder, url)
            

