1.1.3 Update

twint/search.py

-from . import datelock, db, get, feed, output
-from bs4 import BeautifulSoup
-import asyncio
-import concurrent.futures
-import datetime
-import re
-import sys
+from . import datelock, db, get, feed, output, verbose
+from datetime import timedelta
 
 class Search:
     def __init__(self, config):
@@ -12,19 +7,11 @@ class Search:
         self.feed = [-1]
         self.count = 0
         self.config = config
-        self.d = datelock.Set(self.config)
+        self.conn = db.Conn(config.Database)
+        self.d = datelock.Set(self.config.Until, self.config.Since)
+        self.config.TwitterSearch = True
 
-        if self.config.Elasticsearch:
-            print("[+] Indexing to Elasticsearch @ " + str(self.config.Elasticsearch))
-
-        if self.config.Database:
-            print("[+] Inserting into Database: " + str(self.config.Database))
-            self.conn = db.init(self.config.Database)
-            if isinstance(self.conn, str):
-                print(str)
-                sys.exit(1)
-        else:
-            self.conn = ""
+        verbose.Elastic(config)
 
         if not self.config.Timedelta:
             if (self.d._until - self.d._since).days > 30:
@@ -32,12 +19,8 @@ class Search:
             else:
                 self.config.Timedelta = (self.d._until - self.d._since).days
 
-        loop = asyncio.get_event_loop()
-        loop.run_until_complete(self.main())
-
     async def Feed(self):
-        url = await get.Url(self.config, self.init).search()
-        response = await get.Request(self.config, url)
+        response = await get.RequestUrl(self.config, self.init)
         self.feed = []
         try:
             self.feed, self.init = feed.Json(response)
@@ -47,19 +30,7 @@ class Search:
     async def tweets(self):
         await self.Feed()
         if self.config.Location:
-            try:
-                with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
-                    loop = asyncio.get_event_loop()
-                    futures = []
-                    for tweet in self.feed:
-                        self.count += 1
-                        link = tweet.find("a", "tweet-timestamp js-permalink js-nav js-tooltip")["href"]
-                        url = "https://twitter.com{}".format(link)
-                        futures.append(loop.run_in_executor(executor, await get.Tweet(url,
-                            self.config, self.conn)))
-                    await asyncio.gather(*futures)
-            except:
-                pass
+            self.count += await get.Multi(self.feed, self.config, self.conn)
         else:
             for tweet in self.feed:
                 self.count += 1
@@ -67,30 +38,27 @@ class Search:
 
     async def main(self):
         if self.config.User_id is not None:
-            self.config.Username = await get.Username(self.config)
+            self.config.Username = await get.Username(self.config.User_id)
 
         if self.config.Since and self.config.Until:
+            _days = timedelta(days=int(self.config.Timedelta))
             while self.d._since < self.d._until:
-                self.config.Since = str(self.d._until - datetime.timedelta(days=int(self.config.Timedelta)))
+                self.config.Since = str(self.d._until - _days)
                 self.config.Until = str(self.d._until)
                 if len(self.feed) > 0:
                     await self.tweets()
+                elif get.Limit(self.config.Limit, self.count):
+                    self.d._until = self.d._until - _days
                 else:
-                    self.d._until = self.d._until - datetime.timedelta(days=int(self.config.Timedelta))
-                    self.feed = [-1]
-
-                if self.config.Limit is not None and self.count >= int(self.config.Limit):
-                    self.d._until = self.d._until - datetime.timedelta(days=int(self.config.Timedelta))
+                    self.d._until = self.d._until - _days
                     self.feed = [-1]
         else:
             while True:
                 if len(self.feed) > 0:
                     await self.tweets()
-                else:
+                elif get.Limit(self.config.Limit, self.count):
                     break
-
-                if self.config.Limit is not None and self.count >= int(self.config.Limit):
+                else:
                     break
                 
-        if self.config.Count:
-            print("[+] Finished: Successfully collected {0.count} Tweets.".format(self))
+        verbose.Count(self.config, self.count)