diff options
| -rw-r--r-- | comedyGenerator/__init__.py | 0 | ||||
| -rwxr-xr-x | comedyGenerator/comedyGenerator.py (renamed from comedyGenerator) | 66 | ||||
| -rw-r--r-- | contrib/config (renamed from config) | 0 | ||||
| -rw-r--r-- | setup.py | 16 |
4 files changed, 53 insertions, 29 deletions
diff --git a/comedyGenerator/__init__.py b/comedyGenerator/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/comedyGenerator/__init__.py diff --git a/comedyGenerator b/comedyGenerator/comedyGenerator.py index 7cf1190..d3d81aa 100755 --- a/comedyGenerator +++ b/comedyGenerator/comedyGenerator.py @@ -7,7 +7,6 @@ import time import configparser from multiprocessing import Pool import xdg -import configparser import os API_URL = "https://ifunny.co/api/v1/feeds/tags/" @@ -17,6 +16,33 @@ class Video: self.url = url self.name = name +def download(video, name): + if args.verbose: + print("Now running for " + str(video)) + + urlsplit = video.split('.') + if args.verbose: + print("url split = " + str(urlsplit)) + if len(name) > 200: + if args.verbose: + print("shortening name because its too long") + name = name[0:200] + name = name.replace('"', "") + name = name.replace('/', "") + outputPath = args.output + tags + " - " + name + "." + urlsplit[len(urlsplit)-1] + if args.verbose: + print("name read as: " + name) + if os.path.exists(outputPath): + print(name + " already exists!") + else: + print("saving " + video + " as " + name) + if args.type.lower() == "video" or args.type.lower() == "videos": + os.system("ffmpeg -y -i \"" + video + "\" \"" + outputPath + "\"") + elif args.type.lower() == "gifs" or args.type.lower() == "gif": + urllib.request.urlretrieve(video, outputPath) + if args.verbose: + print("saving complete!") + config = configparser.ConfigParser() configdir = str(xdg.XDG_CONFIG_HOME) config.read(configdir + "/comedyGenerator") @@ -154,15 +180,17 @@ for tags in args.tags: for tries in range(100): try: master = requests.get('https://ifunny.co/', headers=headers) - except: + except Exception as error: if tries < 100 - 1: print("Rate Limited! Sleeping for " + str(tries*1.5) + " seconds!") time.sleep(tries*1.5) + print(error) continue break combineHeader = (dict(master.headers)|headers) requestHeader = { - "User-Agent":combineHeader['User-Agent'], + #"User-Agent":combineHeader['User-Agent'], + "User-Agent": "User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0", "Content-Type":combineHeader['Content-Type'], "x-requested-with": "fetch", "x-csrf-token": combineHeader['Set-Cookie'].split(';')[0].split('=')[1], @@ -210,36 +238,16 @@ for tags in args.tags: if args.verbose: print("Videos list truncated! Its now: " + str(len(videos)) + " units long") - def download(video, name): - if args.verbose: - print("Now running for " + str(video)) - - urlsplit = video.split('.') - if args.verbose: - print("url split = " + str(urlsplit)) - if len(name) > 200: - if args.verbose: - print("shortening name because its too long") - name = name[0:200] - name = name.replace('"', "") - name = name.replace('/', "") - outputPath = args.output + tags + " - " + name + "." + urlsplit[len(urlsplit)-1] - if args.verbose: - print("name read as: " + name) - if os.path.exists(outputPath): - print(name + " already exists!") - else: - print("saving " + video + " as " + name) - if args.type.lower() == "video" or args.type.lower() == "videos": - os.system("ffmpeg -y -i \"" + video + "\" \"" + outputPath + "\"") - elif args.type.lower() == "gifs" or args.type.lower() == "gif": - urllib.request.urlretrieve(video, outputPath) - if args.verbose: - print("saving complete!") if args.jobs > 1 and args.noDownload == False: + if args.verbose: + print("Creating multiprocessing pool...") pool = Pool(args.jobs) for video in videos: + if args.verbose: + print("Sending " + video.name + " to multiprocessing pool") pool.apply_async(download, (video.url, video.name,)) + if args.verbose: + print("closing the multiprocessing pool") pool.close() pool.join() elif args.noDownload == False: diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6d2d8e2 --- /dev/null +++ b/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup, find_packages + +setup( + name='comedyGenerator', + version='1.0.3', + description='Ifunny API scrapper for downloading videos and gifs.', + author='msglm', + author_email='msglm@techchud.xyz', + url="https://git.techchud.xyz/comedyGenerator/", + packages=find_packages(), #same as name + install_requires=['ffmpeg', 'xdg'], #external packages as dependencies + entry_points=''' + [console_scripts] + comedyGenerator=comedyGenerator.comedyGenerator:args_parser + ''' + ) |