first

4chanscrape.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+#coding: utf8
+
+#Scraper
+from bs4 import BeautifulSoup
+#Connexion web
+import mechanicalsoup
+import wget
+
+'''
+############## 4chan thread scrapper ################
+Give me a thread, i'll download all of its pictures !
+'''
+
+
+def getArgs():
+    '''Gets all the arguments passed to the script and returns them in a parse_args()-type object.
+    No args
+    Returns:
+    -args : an args object containing all the optional arguments passed to the script.
+    '''
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-u", "--url", help = "URL of the thread", action="store", type=str, required=True)
+    parser.add_argument("-f", "--folder", help = "Folder in which downloads will go", action="store", type=str, required=True)
+    #Creating the args object
+    args=parser.parse_args()
+
+    return args
+
+
+def html_get(url):
+    '''
+    Get html from the webpage
+    Args:
+    - url : a str containing url to scrap
+    Returns:
+    - page.soup: A BeautifulSoup object containing html
+    '''
+    #Browser
+    browser = mechanicalsoup.Browser()
+    try:
+        page = browser.get(url)
+        print(f"Got html from {url}")
+        return page.soup
+    except Exception as e:
+        print(f"Got error {e}")
+
+
+def scraper(soup):
+    '''
+    Scrape a bs4 html object, find posts w/ images and get full size source
+    Args:
+    - soup : bs4 soup item
+    - item : tag to find
+    Returns:
+    - sources : A list of image sources
+    '''
+    tags=soup.findAll('a', {"class": "fileThumb"})
+    sources = []
+    for tag in tags:
+        sources.append(tag['href'])
+
+    #Got image sources, removing any left http headers
+    for index, item in enumerate(sources):
+        sources[index] = item.replace("http://", "")
+        sources[index] = item.replace("https://", "")
+        sources[index] = item.replace("//", "")
+    print(f"{len(sources)} images found")
+    return sources
+
+
+def item_dl(sources, dlfolder):
+    '''
+    Download all items in the sources list to folder dlfolder"
+    Args:
+    - sources : a list of URLs
+    '''
+    for source in sources:
+        fullsource = "http://" + source
+        name = wget.download(fullsource, out=dlfolder)
+        print(f"{name} downloaded")
+
+
+args = getArgs()
+folder = args.folder
+url = args.url
+#url = "https://boards.4chan.org/b/"
+#url = 'https://www.deviantart.com/'
+soup = html_get(url)
+sources = scraper(soup)
+item_dl(sources, folder)

README.md

@@ -0,0 +1,20 @@
+# Scrappers
+
+Two scrappers:
+* The 4chan one dls all images from a thread in best res
+* The other one simply looks for "img" in any given page and downloads images
+
+Install depedencies:
+
+```
+python3 -m pip install beautifulsoup4 mechanicalsoup wget --user
+```
+
+Use:
+
+```
+./4chanscrape.py -u https://boards.4channel.org/c/thread/3846676/gunsmith-cats-thread -f ./downloads
+```
+
+* -u : URL of the page
+* -f : folder where you want to download all pictures

scrape.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+#coding: utf8
+
+#Scraper
+from bs4 import BeautifulSoup
+#Connexion web
+import mechanicalsoup
+import wget
+
+'''
+############## image scrapper ################
+Give me a url, i'll download all of its pictures !
+'''
+
+
+def getArgs():
+    '''Gets all the arguments passed to the script and returns them in a parse_args()-type object.
+    No args
+    Returns:
+    -args : an args object containing all the optional arguments passed to the script.
+    '''
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-u", "--url", help = "URL of the thread", action="store", type=str, required=True)
+    parser.add_argument("-f", "--folder", help = "Folder in which downloads will go", action="store", type=str, required=True)
+    #Creating the args object
+    args=parser.parse_args()
+
+    return args
+
+
+def html_get(url):
+    '''
+    Get html from the webpage
+    Args:
+    - url : a str containing url to scrap
+    Returns:
+    - page.soup: A BeautifulSoup object containing html
+    '''
+    #Browser
+    browser = mechanicalsoup.Browser()
+    try:
+        page = browser.get(url)
+        print(f"Got html from {url}")
+        return page.soup
+    except Exception as e:
+        print(f"Got error {e}")
+
+
+def scraper(soup):
+    '''
+    Scrape a bs4 html object, find posts w/ images and get full size source
+    Args:
+    - soup : bs4 soup item
+    - item : tag to find
+    Returns:
+    - sources : A list of image sources
+    '''
+    tags=soup.findAll('img')
+    sources = []
+    for tag in tags:
+        sources.append(tag['href'])
+
+    #Got image sources, removing any left http headers
+    for index, item in enumerate(sources):
+        sources[index] = item.replace("http://", "")
+        sources[index] = item.replace("https://", "")
+        sources[index] = item.replace("//", "")
+    print(f"{len(sources)} images found")
+    return sources
+
+
+def item_dl(sources, dlfolder):
+    '''
+    Download all items in the sources list to folder dlfolder"
+    Args:
+    - sources : a list of URLs
+    '''
+    for source in sources:
+        fullsource = "http://" + source
+        name = wget.download(fullsource, out=dlfolder)
+        print(f"{name} downloaded")
+
+
+args = getArgs()
+folder = args.folder
+url = args.url
+#url = "https://boards.4chan.org/b/"
+#url = 'https://www.deviantart.com/'
+soup = html_get(url)
+sources = scraper(soup)
+item_dl(sources, folder)