From f4036585629bba17662c88c53855ede5940919c8 Mon Sep 17 00:00:00 2001 From: msglm Date: Sat, 11 May 2024 15:49:17 -0500 Subject: add setup.py support --- comedyGenerator | 275 ----------------------------------- comedyGenerator/__init__.py | 0 comedyGenerator/comedyGenerator.py | 283 +++++++++++++++++++++++++++++++++++++ config | 21 --- contrib/config | 21 +++ setup.py | 16 +++ 6 files changed, 320 insertions(+), 296 deletions(-) delete mode 100755 comedyGenerator create mode 100644 comedyGenerator/__init__.py create mode 100755 comedyGenerator/comedyGenerator.py delete mode 100644 config create mode 100644 contrib/config create mode 100644 setup.py diff --git a/comedyGenerator b/comedyGenerator deleted file mode 100755 index 7cf1190..0000000 --- a/comedyGenerator +++ /dev/null @@ -1,275 +0,0 @@ -#!env python3 -import requests -import urllib.request -import os -import argparse -import time -import configparser -from multiprocessing import Pool -import xdg -import configparser -import os - -API_URL = "https://ifunny.co/api/v1/feeds/tags/" - -class Video: - def __init__(self, url, name): - self.url = url - self.name = name - -config = configparser.ConfigParser() -configdir = str(xdg.XDG_CONFIG_HOME) -config.read(configdir + "/comedyGenerator") - -parser = argparse.ArgumentParser(add_help=True, formatter_class=argparse.RawTextHelpFormatter) - -parser.add_argument('--verbose', '-v', - default=False, - action='store_true', - dest='verbose', - help='Enables verbose output' - ) - -parser.add_argument('--amount','-a', - default=int(config['DEFAULT']['amount']), - dest='amount', - action="store", - help="the amount of funnies you'd like to download per tag", - type=int - ) - -parser.add_argument('--output','-o', - dest=None, - action="store", - help="the output directory of the funnies", - type=str - ) - -parser.add_argument('--log','-l', - default=config['DEFAULT']['log'], - dest='log', - action="store", - help="the output directory of the log file for your funnies. STDOUT if you wish for standard output.", - type=str - ) - -parser.add_argument('--jobs','-j', - default=int(config['DEFAULT']['jobs']), - dest='jobs', - action="store", - help="how many jobs you'd like to use while downloading", - type=int - ) - -parser.add_argument('--no-download', '--nd', - dest='noDownload', - action="store_true", - help="If you would like to actually download the funnies or not", - ) - -parser.add_argument('--type','-t', - default=str(config['DEFAULT']['type']), - dest='type', - action="store", - help= - ''' -The type of content that will be extracted -The following arguments are valid (case insensitive): -Video -Gifs - ''', - type=str - ) - -parser.add_argument('tags', - nargs='+', - type=str, - help='Provides tags to be check for funny downloading' - ) - -args = parser.parse_args() - -contentFilter = "video" - -if args.output == None: - if args.verbose: - print("Recalibtrating output location") - if args.type.lower() == "video" or args.type.lower() == "videos": - args.output = config['Videos']['output'] - elif args.type.lower() == "gifs" or args.type.lower() == "gif": - args.output = config['Gifs']['output'] - -if args.type.lower() == "video" or args.type.lower() == "videos": - contentFilter = "video" -elif args.type.lower() == "gifs" or args.type.lower() == "gif": - contentFilter = "gif" - -if args.verbose: - print("Current output location: " + str(args.output)) - print("Current filter: " + contentFilter) - - -#string comparision so I don't need two vars; bloat kills bloat -logging = False -if not args.log == "False": - logging = True - if args.log == "STDOUT": - logFile = "STDOUT" - else: - logFile = open(args.log, "a") - tmpLog = [] - if args.verbose: - print("We're going to be logging!") - print("Logging Path: " + args.log) - -headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - "DNT": "1", - "Connection": "keep-alive", - "Upgrade-Insecure-Requests": "1", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "cross-site", - "Cache-Control": "max-age=0" - } - -for tags in args.tags: - videos = [] - print("Downloading Tag: " + tags) - headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - "DNT": "1", - "Connection": "keep-alive", - "Upgrade-Insecure-Requests": "1", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "cross-site", - "Cache-Control": "max-age=0" - } - - for tries in range(100): - try: - master = requests.get('https://ifunny.co/', headers=headers) - except: - if tries < 100 - 1: - print("Rate Limited! Sleeping for " + str(tries*1.5) + " seconds!") - time.sleep(tries*1.5) - continue - break - combineHeader = (dict(master.headers)|headers) - requestHeader = { - "User-Agent":combineHeader['User-Agent'], - "Content-Type":combineHeader['Content-Type'], - "x-requested-with": "fetch", - "x-csrf-token": combineHeader['Set-Cookie'].split(';')[0].split('=')[1], - "set-cookies":combineHeader['Set-Cookie'], - "access-control-allow-headers":combineHeader['access-control-allow-headers'] - } - requestCookies = { - "CID" : combineHeader['Set-Cookie'].split(';')[3].split('=')[2], - "sound" : "off", - "viewMode" : "list", - "x-csrf-token": combineHeader['Set-Cookie'].split(';')[0].split('=')[1] - } - for tries in range(100): - try: - tagPage = requests.get(API_URL + tags + "?filter=" + contentFilter, headers=requestHeader, cookies=requestCookies) - if args.verbose: - print("Got Webpage!") - except: - if tries < 100 - 1: - print("Rate Limited! Sleeping for " + str(tries*1.5) + " seconds!") - time.sleep(tries*1.5) - continue - break - - JSONDump = tagPage.json() - - while len(videos) < args.amount and len(JSONDump['items']) > 0: - print("Currently have " + str(len(videos)) + " " + contentFilter + " out of " + str(args.amount) + " (" + str((len(videos)/args.amount)*100) + "%)") - for item in range(len(JSONDump['items'])): - videos.append( Video(JSONDump['items'][item]['url'], JSONDump['items'][item]['title']) ) - for tries in range(100): - try: - tagPage = requests.get(API_URL + tags + "?filter=" + contentFilter + "&next=" + JSONDump['pagination']['next'], headers=requestHeader, cookies=requestCookies) - JSONDump = tagPage.json() - if args.verbose: - print("Got New Tag Page!") - break - except: - if tries < 100 - 1: - print("Rate Limited! Sleeping for " + str(tries*1.5) + " seconds!") - time.sleep(tries*1.5) - continue - if len(videos) > args.amount: - videos = videos[:args.amount] - if args.verbose: - print("Videos list truncated! Its now: " + str(len(videos)) + " units long") - - def download(video, name): - if args.verbose: - print("Now running for " + str(video)) - - urlsplit = video.split('.') - if args.verbose: - print("url split = " + str(urlsplit)) - if len(name) > 200: - if args.verbose: - print("shortening name because its too long") - name = name[0:200] - name = name.replace('"', "") - name = name.replace('/', "") - outputPath = args.output + tags + " - " + name + "." + urlsplit[len(urlsplit)-1] - if args.verbose: - print("name read as: " + name) - if os.path.exists(outputPath): - print(name + " already exists!") - else: - print("saving " + video + " as " + name) - if args.type.lower() == "video" or args.type.lower() == "videos": - os.system("ffmpeg -y -i \"" + video + "\" \"" + outputPath + "\"") - elif args.type.lower() == "gifs" or args.type.lower() == "gif": - urllib.request.urlretrieve(video, outputPath) - if args.verbose: - print("saving complete!") - if args.jobs > 1 and args.noDownload == False: - pool = Pool(args.jobs) - for video in videos: - pool.apply_async(download, (video.url, video.name,)) - pool.close() - pool.join() - elif args.noDownload == False: - for video in videos: - download(video.url, video.name) - - if logging and args.noDownload == False: - for video in videos: - logName = video.url - logPath = args.output + logName - if args.verbose: - print("Writing " + logPath + " to log file") - logFile.write(logPath + "\n") - elif logging: - for video in videos: - logName = video.url - if args.verbose: - print("Writing " + logName + " to log file") - logFile.write(logName + "\n") -if logging: - logFile.close() - -#This program is free software: you can redistribute it and/or modify -#it under the terms of the GNU Affero General Public License version 3 as published by -#the Free Software Foundation. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -#GNU Affero General Public License for more details. -# -#You should have received a copy of the GNU Affero General Public License -#along with this program. If not, see . diff --git a/comedyGenerator/__init__.py b/comedyGenerator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/comedyGenerator/comedyGenerator.py b/comedyGenerator/comedyGenerator.py new file mode 100755 index 0000000..d3d81aa --- /dev/null +++ b/comedyGenerator/comedyGenerator.py @@ -0,0 +1,283 @@ +#!env python3 +import requests +import urllib.request +import os +import argparse +import time +import configparser +from multiprocessing import Pool +import xdg +import os + +API_URL = "https://ifunny.co/api/v1/feeds/tags/" + +class Video: + def __init__(self, url, name): + self.url = url + self.name = name + +def download(video, name): + if args.verbose: + print("Now running for " + str(video)) + + urlsplit = video.split('.') + if args.verbose: + print("url split = " + str(urlsplit)) + if len(name) > 200: + if args.verbose: + print("shortening name because its too long") + name = name[0:200] + name = name.replace('"', "") + name = name.replace('/', "") + outputPath = args.output + tags + " - " + name + "." + urlsplit[len(urlsplit)-1] + if args.verbose: + print("name read as: " + name) + if os.path.exists(outputPath): + print(name + " already exists!") + else: + print("saving " + video + " as " + name) + if args.type.lower() == "video" or args.type.lower() == "videos": + os.system("ffmpeg -y -i \"" + video + "\" \"" + outputPath + "\"") + elif args.type.lower() == "gifs" or args.type.lower() == "gif": + urllib.request.urlretrieve(video, outputPath) + if args.verbose: + print("saving complete!") + +config = configparser.ConfigParser() +configdir = str(xdg.XDG_CONFIG_HOME) +config.read(configdir + "/comedyGenerator") + +parser = argparse.ArgumentParser(add_help=True, formatter_class=argparse.RawTextHelpFormatter) + +parser.add_argument('--verbose', '-v', + default=False, + action='store_true', + dest='verbose', + help='Enables verbose output' + ) + +parser.add_argument('--amount','-a', + default=int(config['DEFAULT']['amount']), + dest='amount', + action="store", + help="the amount of funnies you'd like to download per tag", + type=int + ) + +parser.add_argument('--output','-o', + dest=None, + action="store", + help="the output directory of the funnies", + type=str + ) + +parser.add_argument('--log','-l', + default=config['DEFAULT']['log'], + dest='log', + action="store", + help="the output directory of the log file for your funnies. STDOUT if you wish for standard output.", + type=str + ) + +parser.add_argument('--jobs','-j', + default=int(config['DEFAULT']['jobs']), + dest='jobs', + action="store", + help="how many jobs you'd like to use while downloading", + type=int + ) + +parser.add_argument('--no-download', '--nd', + dest='noDownload', + action="store_true", + help="If you would like to actually download the funnies or not", + ) + +parser.add_argument('--type','-t', + default=str(config['DEFAULT']['type']), + dest='type', + action="store", + help= + ''' +The type of content that will be extracted +The following arguments are valid (case insensitive): +Video +Gifs + ''', + type=str + ) + +parser.add_argument('tags', + nargs='+', + type=str, + help='Provides tags to be check for funny downloading' + ) + +args = parser.parse_args() + +contentFilter = "video" + +if args.output == None: + if args.verbose: + print("Recalibtrating output location") + if args.type.lower() == "video" or args.type.lower() == "videos": + args.output = config['Videos']['output'] + elif args.type.lower() == "gifs" or args.type.lower() == "gif": + args.output = config['Gifs']['output'] + +if args.type.lower() == "video" or args.type.lower() == "videos": + contentFilter = "video" +elif args.type.lower() == "gifs" or args.type.lower() == "gif": + contentFilter = "gif" + +if args.verbose: + print("Current output location: " + str(args.output)) + print("Current filter: " + contentFilter) + + +#string comparision so I don't need two vars; bloat kills bloat +logging = False +if not args.log == "False": + logging = True + if args.log == "STDOUT": + logFile = "STDOUT" + else: + logFile = open(args.log, "a") + tmpLog = [] + if args.verbose: + print("We're going to be logging!") + print("Logging Path: " + args.log) + +headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "DNT": "1", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "cross-site", + "Cache-Control": "max-age=0" + } + +for tags in args.tags: + videos = [] + print("Downloading Tag: " + tags) + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "DNT": "1", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "cross-site", + "Cache-Control": "max-age=0" + } + + for tries in range(100): + try: + master = requests.get('https://ifunny.co/', headers=headers) + except Exception as error: + if tries < 100 - 1: + print("Rate Limited! Sleeping for " + str(tries*1.5) + " seconds!") + time.sleep(tries*1.5) + print(error) + continue + break + combineHeader = (dict(master.headers)|headers) + requestHeader = { + #"User-Agent":combineHeader['User-Agent'], + "User-Agent": "User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0", + "Content-Type":combineHeader['Content-Type'], + "x-requested-with": "fetch", + "x-csrf-token": combineHeader['Set-Cookie'].split(';')[0].split('=')[1], + "set-cookies":combineHeader['Set-Cookie'], + "access-control-allow-headers":combineHeader['access-control-allow-headers'] + } + requestCookies = { + "CID" : combineHeader['Set-Cookie'].split(';')[3].split('=')[2], + "sound" : "off", + "viewMode" : "list", + "x-csrf-token": combineHeader['Set-Cookie'].split(';')[0].split('=')[1] + } + for tries in range(100): + try: + tagPage = requests.get(API_URL + tags + "?filter=" + contentFilter, headers=requestHeader, cookies=requestCookies) + if args.verbose: + print("Got Webpage!") + except: + if tries < 100 - 1: + print("Rate Limited! Sleeping for " + str(tries*1.5) + " seconds!") + time.sleep(tries*1.5) + continue + break + + JSONDump = tagPage.json() + + while len(videos) < args.amount and len(JSONDump['items']) > 0: + print("Currently have " + str(len(videos)) + " " + contentFilter + " out of " + str(args.amount) + " (" + str((len(videos)/args.amount)*100) + "%)") + for item in range(len(JSONDump['items'])): + videos.append( Video(JSONDump['items'][item]['url'], JSONDump['items'][item]['title']) ) + for tries in range(100): + try: + tagPage = requests.get(API_URL + tags + "?filter=" + contentFilter + "&next=" + JSONDump['pagination']['next'], headers=requestHeader, cookies=requestCookies) + JSONDump = tagPage.json() + if args.verbose: + print("Got New Tag Page!") + break + except: + if tries < 100 - 1: + print("Rate Limited! Sleeping for " + str(tries*1.5) + " seconds!") + time.sleep(tries*1.5) + continue + if len(videos) > args.amount: + videos = videos[:args.amount] + if args.verbose: + print("Videos list truncated! Its now: " + str(len(videos)) + " units long") + + if args.jobs > 1 and args.noDownload == False: + if args.verbose: + print("Creating multiprocessing pool...") + pool = Pool(args.jobs) + for video in videos: + if args.verbose: + print("Sending " + video.name + " to multiprocessing pool") + pool.apply_async(download, (video.url, video.name,)) + if args.verbose: + print("closing the multiprocessing pool") + pool.close() + pool.join() + elif args.noDownload == False: + for video in videos: + download(video.url, video.name) + + if logging and args.noDownload == False: + for video in videos: + logName = video.url + logPath = args.output + logName + if args.verbose: + print("Writing " + logPath + " to log file") + logFile.write(logPath + "\n") + elif logging: + for video in videos: + logName = video.url + if args.verbose: + print("Writing " + logName + " to log file") + logFile.write(logName + "\n") +if logging: + logFile.close() + +#This program is free software: you can redistribute it and/or modify +#it under the terms of the GNU Affero General Public License version 3 as published by +#the Free Software Foundation. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#GNU Affero General Public License for more details. +# +#You should have received a copy of the GNU Affero General Public License +#along with this program. If not, see . diff --git a/config b/config deleted file mode 100644 index c6669f7..0000000 --- a/config +++ /dev/null @@ -1,21 +0,0 @@ -#Move this to ~/.config/comedyGenerator -[DEFAULT] - -#Location of the log file, if you care about that. False will disable the log file -log=False - -#Default amount of funnies to download -amount=10 - -#Default amount of jobs used when downloading -jobs=1 - -[Videos] - -#where videos will be outputted to -output=/home/msglm/Videos/unsorted/ - -[Gifs] - -#where gifs will be outputted to -output=/home/msglm/Pictures/gif/ diff --git a/contrib/config b/contrib/config new file mode 100644 index 0000000..c6669f7 --- /dev/null +++ b/contrib/config @@ -0,0 +1,21 @@ +#Move this to ~/.config/comedyGenerator +[DEFAULT] + +#Location of the log file, if you care about that. False will disable the log file +log=False + +#Default amount of funnies to download +amount=10 + +#Default amount of jobs used when downloading +jobs=1 + +[Videos] + +#where videos will be outputted to +output=/home/msglm/Videos/unsorted/ + +[Gifs] + +#where gifs will be outputted to +output=/home/msglm/Pictures/gif/ diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6d2d8e2 --- /dev/null +++ b/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup, find_packages + +setup( + name='comedyGenerator', + version='1.0.3', + description='Ifunny API scrapper for downloading videos and gifs.', + author='msglm', + author_email='msglm@techchud.xyz', + url="https://git.techchud.xyz/comedyGenerator/", + packages=find_packages(), #same as name + install_requires=['ffmpeg', 'xdg'], #external packages as dependencies + entry_points=''' + [console_scripts] + comedyGenerator=comedyGenerator.comedyGenerator:args_parser + ''' + ) -- cgit v1.2.3