summaryrefslogtreecommitdiffstats
path: root/aux-files/snscrape
diff options
context:
space:
mode:
Diffstat (limited to 'aux-files/snscrape')
-rw-r--r--aux-files/snscrape/snscrape-downloads-telegram.patch495
1 files changed, 495 insertions, 0 deletions
diff --git a/aux-files/snscrape/snscrape-downloads-telegram.patch b/aux-files/snscrape/snscrape-downloads-telegram.patch
new file mode 100644
index 0000000..46665c4
--- /dev/null
+++ b/aux-files/snscrape/snscrape-downloads-telegram.patch
@@ -0,0 +1,495 @@
+From 00239388e3096277a55271a8786b4b5d6d2bec84 Mon Sep 17 00:00:00 2001
+From: John O'Sullivan <j.osullivan42@gmail.com>
+Date: Thu, 18 Jan 2024 11:37:32 -0500
+Subject: [PATCH 1/8] WIP: Fixed 2.5 out of 5 issues mentioned in PR
+
+---
+ snscrape/base.py | 1 +
+ snscrape/modules/telegram.py | 12 ++++++------
+ 2 files changed, 7 insertions(+), 6 deletions(-)
+
+diff --git a/snscrape/base.py b/snscrape/base.py
+index c9e75d9d..5ce5e1da 100644
+--- a/snscrape/base.py
++++ b/snscrape/base.py
+@@ -193,6 +193,7 @@ def _request(self, method, url, params = None, data = None, headers = None, time
+ # The request is newly prepared on each retry because of potential cookie updates.
+ req = self._session.prepare_request(requests.Request(method, url, params = params, data = data, headers = headers))
+ environmentSettings = self._session.merge_environment_settings(req.url, proxies, None, None, None)
++ _logger.info("Hey there, I'm in here")
+ _logger.info(f'Retrieving {req.url}')
+ _logger.debug(f'... with headers: {headers!r}')
+ if data:
+diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py
+index 4e977656..54345d96 100644
+--- a/snscrape/modules/telegram.py
++++ b/snscrape/modules/telegram.py
+@@ -196,6 +196,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False):
+ }
+ timeTag = videoPlayer.find('time')
+ if timeTag is None:
++ _logger.warning(f'Could not find duration for video or GIF at {url}')
+ cls = Gif
+ else:
+ cls = Video
+@@ -219,8 +220,6 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False):
+ else:
+ _logger.warning(f'Could not process link preview image on {url}')
+ linkPreview = LinkPreview(**kwargs)
+- if kwargs['href'] in outlinks:
+- outlinks.remove(kwargs['href'])
+
+ viewsSpan = post.find('span', class_ = 'tgme_widget_message_views')
+ views = None if viewsSpan is None else _parse_num(viewsSpan.text)
+@@ -239,13 +238,14 @@ def get_items(self):
+ return
+ nextPageUrl = ''
+ while True:
++ print("About to yield from get_items")
+ yield from self._soup_to_items(soup, r.url)
+- try:
+- if soup.find('a', attrs = {'class': 'tgme_widget_message_date'}, href = True)['href'].split('/')[-1] == '1':
++ dateElt = soup.find('a', attrs = {'class': 'tgme_widget_message_date'}, href = True)
++ if dateElt and 'href' in dateElt.attrs:
++ urlPieces = dateElt['href'].split('/')
++ if urlPieces and urlPieces[-1] == '1':
+ # if message 1 is the first message in the page, terminate scraping
+ break
+- except:
+- pass
+ pageLink = soup.find('a', attrs = {'class': 'tme_messages_more', 'data-before': True})
+ if not pageLink:
+ # some pages are missing a "tme_messages_more" tag, causing early termination
+
+From 670905fedb64656b94c6fb920c8628d318171b64 Mon Sep 17 00:00:00 2001
+From: John O'Sullivan <j.osullivan42@gmail.com>
+Date: Thu, 18 Jan 2024 11:46:46 -0500
+Subject: [PATCH 2/8] Remove test log statement, add link to example GIF
+
+---
+ snscrape/base.py | 1 -
+ snscrape/modules/telegram.py | 2 +-
+ 2 files changed, 1 insertion(+), 2 deletions(-)
+
+diff --git a/snscrape/base.py b/snscrape/base.py
+index 5ce5e1da..c9e75d9d 100644
+--- a/snscrape/base.py
++++ b/snscrape/base.py
+@@ -193,7 +193,6 @@ def _request(self, method, url, params = None, data = None, headers = None, time
+ # The request is newly prepared on each retry because of potential cookie updates.
+ req = self._session.prepare_request(requests.Request(method, url, params = params, data = data, headers = headers))
+ environmentSettings = self._session.merge_environment_settings(req.url, proxies, None, None, None)
+- _logger.info("Hey there, I'm in here")
+ _logger.info(f'Retrieving {req.url}')
+ _logger.debug(f'... with headers: {headers!r}')
+ if data:
+diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py
+index 54345d96..01e99318 100644
+--- a/snscrape/modules/telegram.py
++++ b/snscrape/modules/telegram.py
+@@ -196,7 +196,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False):
+ }
+ timeTag = videoPlayer.find('time')
+ if timeTag is None:
+- _logger.warning(f'Could not find duration for video or GIF at {url}')
++ # Example of duration-less GIF: https://t.me/thisisatestchannel19451923/3
+ cls = Gif
+ else:
+ cls = Video
+
+From 54df8832f5b5bc3af58c3faf953966a2070a834d Mon Sep 17 00:00:00 2001
+From: John O'Sullivan <j.osullivan42@gmail.com>
+Date: Thu, 22 Feb 2024 01:06:04 -0500
+Subject: [PATCH 3/8] Added media processing into main link loop; using prev
+ tag to get page, rather than index math
+
+---
+ snscrape/modules/telegram.py | 84 +++++++++++++++++++-----------------
+ 1 file changed, 44 insertions(+), 40 deletions(-)
+
+diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py
+index 01e99318..b4f3d78e 100644
+--- a/snscrape/modules/telegram.py
++++ b/snscrape/modules/telegram.py
+@@ -152,7 +152,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False):
+ imageUrls = _STYLE_MEDIA_URL_PATTERN.findall(style)
+ if len(imageUrls) == 1:
+ media.append(Photo(url = imageUrls[0]))
+- continue
++
+ if _SINGLE_MEDIA_LINK_PATTERN.match(link['href']):
+ style = link.attrs.get('style', '')
+ imageUrls = _STYLE_MEDIA_URL_PATTERN.findall(style)
+@@ -161,49 +161,23 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False):
+ # resp = self._get(image[0])
+ # encoded_string = base64.b64encode(resp.content)
+ # Individual photo or video link
+- continue
++
+ if link.text.startswith('@'):
+ mentions.append(link.text.strip('@'))
+- continue
++
+ if link.text.startswith('#'):
+ hashtags.append(link.text.strip('#'))
+- continue
++
++ if 'tgme_widget_message_voice_player' in link.get('class', []):
++ media.append(_parse_voice_message(link))
++
++ if 'tgme_widget_message_video_player' in link.get('class', []):
++ media.append(_parse_video_message(link))
++
+ href = urllib.parse.urljoin(pageUrl, link['href'])
+ if (href not in outlinks) and (href != rawUrl) and (href != forwardedUrl):
+ outlinks.append(href)
+
+- for voicePlayer in post.find_all('a', {'class': 'tgme_widget_message_voice_player'}):
+- audioUrl = voicePlayer.find('audio')['src']
+- durationStr = voicePlayer.find('time').text
+- duration = _durationStrToSeconds(durationStr)
+- barHeights = [float(s['style'].split(':')[-1].strip(';%')) for s in voicePlayer.find('div', {'class': 'bar'}).find_all('s')]
+-
+- media.append(VoiceMessage(url = audioUrl, duration = duration, bars = barHeights))
+-
+- for videoPlayer in post.find_all('a', {'class': 'tgme_widget_message_video_player'}):
+- iTag = videoPlayer.find('i')
+- if iTag is None:
+- videoUrl = None
+- videoThumbnailUrl = None
+- else:
+- style = iTag['style']
+- videoThumbnailUrl = _STYLE_MEDIA_URL_PATTERN.findall(style)[0]
+- videoTag = videoPlayer.find('video')
+- videoUrl = None if videoTag is None else videoTag['src']
+- mKwargs = {
+- 'thumbnailUrl': videoThumbnailUrl,
+- 'url': videoUrl,
+- }
+- timeTag = videoPlayer.find('time')
+- if timeTag is None:
+- # Example of duration-less GIF: https://t.me/thisisatestchannel19451923/3
+- cls = Gif
+- else:
+- cls = Video
+- durationStr = videoPlayer.find('time').text
+- mKwargs['duration'] = _durationStrToSeconds(durationStr)
+- media.append(cls(**mKwargs))
+-
+ linkPreview = None
+ if (linkPreviewA := post.find('a', class_ = 'tgme_widget_message_link_preview')):
+ kwargs = {}
+@@ -250,10 +224,10 @@ def get_items(self):
+ if not pageLink:
+ # some pages are missing a "tme_messages_more" tag, causing early termination
+ if '=' not in nextPageUrl:
+- nextPageUrl = soup.find('link', attrs = {'rel': 'canonical'}, href = True)['href']
+- nextPostIndex = int(nextPageUrl.split('=')[-1]) - 20
++ nextPageUrl = soup.find('link', attrs = {'rel': 'prev'}, href = True)['href']
++ nextPostIndex = int(nextPageUrl.split('=')[-1])
+ if nextPostIndex > 20:
+- pageLink = {'href': nextPageUrl.split('=')[0] + f'={nextPostIndex}'}
++ pageLink = {'href': nextPageUrl}
+ else:
+ break
+ nextPageUrl = urllib.parse.urljoin(r.url, pageLink['href'])
+@@ -333,4 +307,34 @@ def _telegramResponseOkCallback(r):
+ if r.status_code == 200:
+ return (True, None)
+ return (False, f'{r.status_code=}')
+-
+\ No newline at end of file
++
++def _parse_voice_message(voicePlayer):
++ audioUrl = voicePlayer.find('audio')['src']
++ durationStr = voicePlayer.find('time').text
++ duration = _durationStrToSeconds(durationStr)
++ barHeights = [float(s['style'].split(':')[-1].strip(';%')) for s in voicePlayer.find('div', {'class': 'bar'}).find_all('s')]
++ return VoiceMessage(url = audioUrl, duration = duration, bars = barHeights)
++
++def _parse_video_message(videoPlayer):
++ iTag = videoPlayer.find('i')
++ if iTag is None:
++ videoUrl = None
++ videoThumbnailUrl = None
++ else:
++ style = iTag['style']
++ videoThumbnailUrl = _STYLE_MEDIA_URL_PATTERN.findall(style)[0]
++ videoTag = videoPlayer.find('video')
++ videoUrl = None if videoTag is None else videoTag['src']
++ mKwargs = {
++ 'thumbnailUrl': videoThumbnailUrl,
++ 'url': videoUrl,
++ }
++ timeTag = videoPlayer.find('time')
++ if timeTag is None:
++ # Example of duration-less GIF: https://t.me/thisisatestchannel19451923/3
++ cls = Gif
++ else:
++ cls = Video
++ durationStr = videoPlayer.find('time').text
++ mKwargs['duration'] = _durationStrToSeconds(durationStr)
++ return cls(**mKwargs)
+\ No newline at end of file
+
+From 2dfd1542f19bbadad603e00e61712943542fbfe1 Mon Sep 17 00:00:00 2001
+From: John O'Sullivan <j.osullivan42@gmail.com>
+Date: Thu, 22 Feb 2024 01:07:46 -0500
+Subject: [PATCH 4/8] Forgot to remove a test log
+
+---
+ snscrape/modules/telegram.py | 1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py
+index b4f3d78e..8f6d18d7 100644
+--- a/snscrape/modules/telegram.py
++++ b/snscrape/modules/telegram.py
+@@ -212,7 +212,6 @@ def get_items(self):
+ return
+ nextPageUrl = ''
+ while True:
+- print("About to yield from get_items")
+ yield from self._soup_to_items(soup, r.url)
+ dateElt = soup.find('a', attrs = {'class': 'tgme_widget_message_date'}, href = True)
+ if dateElt and 'href' in dateElt.attrs:
+
+From a93f6a3fad0d19209a49c7b730fea73659743774 Mon Sep 17 00:00:00 2001
+From: John O'Sullivan <j.osullivan42@gmail.com>
+Date: Fri, 1 Mar 2024 12:51:26 -0500
+Subject: [PATCH 5/8] Applying trislee's suggested fix for getting nextPageUrl
+
+---
+ snscrape/modules/telegram.py | 16 +++++-----------
+ 1 file changed, 5 insertions(+), 11 deletions(-)
+
+diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py
+index 8f6d18d7..ac0feef8 100644
+--- a/snscrape/modules/telegram.py
++++ b/snscrape/modules/telegram.py
+@@ -219,17 +219,11 @@ def get_items(self):
+ if urlPieces and urlPieces[-1] == '1':
+ # if message 1 is the first message in the page, terminate scraping
+ break
+- pageLink = soup.find('a', attrs = {'class': 'tme_messages_more', 'data-before': True})
+- if not pageLink:
+- # some pages are missing a "tme_messages_more" tag, causing early termination
+- if '=' not in nextPageUrl:
+- nextPageUrl = soup.find('link', attrs = {'rel': 'prev'}, href = True)['href']
+- nextPostIndex = int(nextPageUrl.split('=')[-1])
+- if nextPostIndex > 20:
+- pageLink = {'href': nextPageUrl}
+- else:
+- break
+- nextPageUrl = urllib.parse.urljoin(r.url, pageLink['href'])
++ if pageLink := soup.find('link', attrs = {'rel': 'prev'}, href = True):
++ nextPageUrl = urllib.parse.urljoin(r.url, pageLink['href'])
++ else:
++ nextPostIndex = int(soup.find('div', attrs = {'class': 'tgme_widget_message', 'data-post': True})["data-post"].split("/")[-1])
++ nextPageUrl = urllib.parse.urljoin(r.url, r.url.split('?')[0] + f'?before={nextPostIndex}')
+ r = self._get(nextPageUrl, headers = self._headers, responseOkCallback = _telegramResponseOkCallback)
+ if r.status_code != 200:
+ raise snscrape.base.ScraperException(f'Got status code {r.status_code}')
+
+From a542aa57598f94f69fd7b69789e97045e92133da Mon Sep 17 00:00:00 2001
+From: John O'Sullivan <j.osullivan42@gmail.com>
+Date: Thu, 14 Mar 2024 01:50:38 -0400
+Subject: [PATCH 6/8] Ensured termination on channels w/o an id=1 post, wrote
+ test cases to prevent regression
+
+---
+ snscrape/modules/telegram.py | 87 +++++++++++++++++++++++++++++++++++-
+ 1 file changed, 86 insertions(+), 1 deletion(-)
+
+diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py
+index ac0feef8..7a85cb58 100644
+--- a/snscrape/modules/telegram.py
++++ b/snscrape/modules/telegram.py
+@@ -9,6 +9,8 @@
+ import snscrape.base
+ import typing
+ import urllib.parse
++import unittest
++import threading
+
+ _logger = logging.getLogger(__name__)
+ _SINGLE_MEDIA_LINK_PATTERN = re.compile(r'^https://t\.me/[^/]+/\d+\?single$')
+@@ -212,6 +214,8 @@ def get_items(self):
+ return
+ nextPageUrl = ''
+ while True:
++ if soup.find("div", class_ = "tme_no_messages_found"):
++ break
+ yield from self._soup_to_items(soup, r.url)
+ dateElt = soup.find('a', attrs = {'class': 'tgme_widget_message_date'}, href = True)
+ if dateElt and 'href' in dateElt.attrs:
+@@ -330,4 +334,85 @@ def _parse_video_message(videoPlayer):
+ cls = Video
+ durationStr = videoPlayer.find('time').text
+ mKwargs['duration'] = _durationStrToSeconds(durationStr)
+- return cls(**mKwargs)
+\ No newline at end of file
++ return cls(**mKwargs)
++
++class TestTelegramChannelScraper(unittest.TestCase):
++
++ @staticmethod
++ def execute_with_timeout(func, timeout=10):
++ """
++ Executes a function in a separate thread and enforces a timeout.
++ If provided function throws an error, it's re-raised in main thread.
++ Used to detect infinite loops in finite time, works cross-platform.
++
++ :param func: The function to execute. This function should accept no arguments.
++ :param timeout: The timeout in seconds.
++ """
++ exceptions=[]
++ def func_passing_exceptions():
++ try:
++ func()
++ except Exception as e:
++ exceptions.append((e.__class__, e, e.__traceback__))
++
++ thread = threading.Thread(target=func_passing_exceptions)
++ thread.start()
++ thread.join(timeout=timeout)
++
++ if exceptions:
++ exc_class, exc_instance, traceback = exceptions[0]
++ raise exc_class(exc_instance).with_traceback(traceback)
++
++ if thread.is_alive():
++ raise TimeoutError(f"Function didn't complete within {timeout} seconds")
++
++ def test_scraping_termination_missing_prev(self):
++ """Test scraping always terminates, even if the page's prev link is missing."""
++
++ def scrape_two_pages():
++ scraper = TelegramChannelScraper('WLM_USA_TEXAS?before=3766')
++ items = list()
++ num_items_on_page = 20
++ for item in scraper.get_items():
++ items.append(item)
++ if len(items) > 2 * num_items_on_page:
++ break
++
++ self.execute_with_timeout(scrape_two_pages)
++
++ def test_scraping_termination_small_post_count(self):
++ """Test scraping always terminates, even with small number of posts. This channel has only 28."""
++
++ def scrape_small_channel():
++ scraper = TelegramChannelScraper('AKCPB')
++ items = list(scraper.get_items())
++ return items
++
++ self.execute_with_timeout(scrape_small_channel)
++
++ def test_scraping_termination_channels_without_post_id_one(self):
++ """Test scraping gracefully handles channels missing a post where id=1."""
++
++ def scrape_empty_page():
++ scraper = TelegramChannelScraper('BREAKDCODE?before=3')
++ for _ in scraper.get_items():
++ pass
++
++ self.execute_with_timeout(scrape_empty_page)
++
++ def test_media_order_preservation(self):
++ """Test scraped media appears in the same order as in the post."""
++ scraper = TelegramChannelScraper('nexta_live?before=43103')
++ item = next(scraper.get_items(), None)
++ self.assertIsNotNone(item, "Failed to scrape any posts.")
++ self.assertEqual(item.url, "https://t.me/s/nexta_live/43102")
++
++ # Directly validate the types of the objects in the media array
++ expected_types = [Video, Photo, Video] # Adjust based on expected types
++ actual_types = [type(media) for media in item.media] if item.media else []
++
++ self.assertEqual(actual_types, expected_types, "Media did not appear in the expected order.")
++
++
++if __name__ == '__main__':
++ unittest.main()
+
+From 7d061cb5279e153f829340f848bc4ba01d716f26 Mon Sep 17 00:00:00 2001
+From: John O'Sullivan <j.osullivan42@gmail.com>
+Date: Thu, 14 Mar 2024 01:55:16 -0400
+Subject: [PATCH 7/8] Add docstring saying suite should run by directly running
+ file
+
+---
+ snscrape/modules/telegram.py | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py
+index 7a85cb58..c6e0b0ee 100644
+--- a/snscrape/modules/telegram.py
++++ b/snscrape/modules/telegram.py
+@@ -336,7 +336,9 @@ def _parse_video_message(videoPlayer):
+ mKwargs['duration'] = _durationStrToSeconds(durationStr)
+ return cls(**mKwargs)
+
++
+ class TestTelegramChannelScraper(unittest.TestCase):
++ """Run suite by directly calling this file."""
+
+ @staticmethod
+ def execute_with_timeout(func, timeout=10):
+
+From 9309b1b01c6db15862809623e2c5adddecd894be Mon Sep 17 00:00:00 2001
+From: John O'Sullivan <j.osullivan42@gmail.com>
+Date: Thu, 14 Mar 2024 02:00:50 -0400
+Subject: [PATCH 8/8] Correct some inaccurate test descriptions
+
+---
+ snscrape/modules/telegram.py | 13 +++++++------
+ 1 file changed, 7 insertions(+), 6 deletions(-)
+
+diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py
+index c6e0b0ee..dbf0f9b3 100644
+--- a/snscrape/modules/telegram.py
++++ b/snscrape/modules/telegram.py
+@@ -338,7 +338,7 @@ def _parse_video_message(videoPlayer):
+
+
+ class TestTelegramChannelScraper(unittest.TestCase):
+- """Run suite by directly calling this file."""
++ """Run suite by directly running this file."""
+
+ @staticmethod
+ def execute_with_timeout(func, timeout=10):
+@@ -383,7 +383,7 @@ def scrape_two_pages():
+ self.execute_with_timeout(scrape_two_pages)
+
+ def test_scraping_termination_small_post_count(self):
+- """Test scraping always terminates, even with small number of posts. This channel has only 28."""
++ """Test scraping always terminates, even with small number of posts. This channel's highest ID is 28."""
+
+ def scrape_small_channel():
+ scraper = TelegramChannelScraper('AKCPB')
+@@ -392,8 +392,8 @@ def scrape_small_channel():
+
+ self.execute_with_timeout(scrape_small_channel)
+
+- def test_scraping_termination_channels_without_post_id_one(self):
+- """Test scraping gracefully handles channels missing a post where id=1."""
++ def test_scraping_termination_pages_without_posts(self):
++ """Test scraping gracefully handles pages without any posts."""
+
+ def scrape_empty_page():
+ scraper = TelegramChannelScraper('BREAKDCODE?before=3')
+@@ -407,10 +407,11 @@ def test_media_order_preservation(self):
+ scraper = TelegramChannelScraper('nexta_live?before=43103')
+ item = next(scraper.get_items(), None)
+ self.assertIsNotNone(item, "Failed to scrape any posts.")
++
++ # This particular post is known to include media [Video, Photo, Video]
+ self.assertEqual(item.url, "https://t.me/s/nexta_live/43102")
+
+- # Directly validate the types of the objects in the media array
+- expected_types = [Video, Photo, Video] # Adjust based on expected types
++ expected_types = [Video, Photo, Video]
+ actual_types = [type(media) for media in item.media] if item.media else []
+
+ self.assertEqual(actual_types, expected_types, "Media did not appear in the expected order.")
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-01 08:28:20 +0000