|
|
@ -6,7 +6,7 @@ from bs4 import BeautifulSoup |
|
|
|
import enum |
|
|
|
from postData import PostData |
|
|
|
from prepOutput import outputSelect |
|
|
|
from funcs import getThread, stop_page, getFilter, reactionSelect, getRatingsThreshold |
|
|
|
from funcs import getThread, stop_page, getFilter, reactionSelect, getRatingsThreshold, getUsername |
|
|
|
from collectPosts import collectPosts |
|
|
|
|
|
|
|
class Filter(enum.Enum):#enumeration for type of filter |
|
|
@ -17,6 +17,7 @@ class Filter(enum.Enum):#enumeration for type of filter |
|
|
|
total = 4 #total ratings |
|
|
|
specific = 5 #specific rating |
|
|
|
weighted = 6 #weighted rating |
|
|
|
user = 7 #specific user |
|
|
|
|
|
|
|
###### Variables ###### |
|
|
|
#thread data |
|
|
@ -25,6 +26,7 @@ threadTitle = ""#title of thread |
|
|
|
pageData = ""#BeautifulSoup object storing threadpage's HTML |
|
|
|
savedPosts = [] #stores PostData objects for posts to be saved |
|
|
|
pageNum = 0 #current page number |
|
|
|
usernameFilter = "" #username to filter by when applicable |
|
|
|
|
|
|
|
#filter toggles |
|
|
|
filterSelection = Filter.NA #type of filter being used |
|
|
@ -54,12 +56,15 @@ stopPage = stop_page() |
|
|
|
filterSelection = Filter(getFilter()) |
|
|
|
|
|
|
|
if filterSelection == Filter.specific:#if filtering by specific rating ask which rating to use |
|
|
|
ratingSelection = reactionSelect() |
|
|
|
ratingSelection = reactionSelect() |
|
|
|
|
|
|
|
#colect ratings threshold |
|
|
|
#if filtering by specific user, collect username |
|
|
|
if filterSelection == Filter.user: usernameFilter = getUsername() |
|
|
|
|
|
|
|
#colect ratings threshold if user is filtering by ratings |
|
|
|
print("------------------------------------------") |
|
|
|
if filterSelection == Filter.specific: minRating = getRatingsThreshold(filterSelection, ratingSelection) |
|
|
|
else: minRating = getRatingsThreshold(filterSelection) |
|
|
|
elif not filterSelection == Filter.user: minRating = getRatingsThreshold(filterSelection) |
|
|
|
|
|
|
|
#get current page number |
|
|
|
try: pageNum = int(pageData.find("li", class_ = "pageNav-page--current").find("a").text) |
|
|
@ -67,7 +72,7 @@ except: pageNum = 1 |
|
|
|
|
|
|
|
#scrape for posts |
|
|
|
print("\nGrab some popcorn, this might take a while...\n") |
|
|
|
savedPosts = collectPosts(pageData = pageData, filter = filterSelection, minRating = minRating, pageNum = pageNum, stopPage = stopPage, ratingFilter=ratingSelection) |
|
|
|
savedPosts = collectPosts(pageData = pageData, filter = filterSelection, pageNum = pageNum, minRating = minRating, user = usernameFilter, stopPage = stopPage, ratingFilter=ratingSelection) |
|
|
|
|
|
|
|
# ask user where to save json file and what to name it |
|
|
|
print("------------------------------------------") |
|
|
|