From ca78d58458edeb5fbc0e4a94b5eb4bf36a731139 Mon Sep 17 00:00:00 2001 From: justine Date: Thu, 12 Aug 2021 00:54:29 +0000 Subject: [PATCH] Update 'scrape.py' Fixed dedupe --- scrape.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrape.py b/scrape.py index 3e1f033..862dea2 100755 --- a/scrape.py +++ b/scrape.py @@ -84,6 +84,8 @@ def item_dl(sources, dlfolder): for source in sources: fullsource = "http://" + source + if dlfolder[-1] == "/": + dlfolder = dlfolder[:-1] if not deduplicate(fullsource, dlfolder.split("/")[-1]): name = wget.download(fullsource, out=dlfolder) print(f"{name} downloaded")