Lookup patch (#873)

* favorite and following functions are working again using the user_id that fetched with tweet search function, also removed some of the user agents that returns dynamic site which is an undesired situation

* lookup url updated

* bio, location, url parsing ok

* returning join date empty since its not coming with the new interface

* inf function updated to new interface; name, username and private status fully ok, user_id will be parsed if there is a mention of the account itself on the first page

* stat function ok for tweets, following and followers counts. Favorite and media counts do not exist on the profile page

* verified function updated to new page

* updated User function, media count, likes, background image link are empty since they are not included in the new one
Co-authored-by: yunusemrecatalcam <yunus.catalcam@gmail.com>

twint/run.py

@@ -6,6 +6,8 @@ from .storage import db
 
 import logging as logme
 
+import time
+
 class Twint:
     def __init__(self, config):
         logme.debug(__name__+':Twint:__init__')
@@ -348,7 +350,7 @@ def Lookup(config):
             logme.debug(__name__+':Twint:Lookup:user_id')
             config.Username = get_event_loop().run_until_complete(get.Username(config.User_id))
 
-        url = f"https://twitter.com/{config.Username}?lang=en"
+        url = f"https://mobile.twitter.com/{config.Username}?prefetchTimestamp=" + str(int(time.time() * 1000))
         get_event_loop().run_until_complete(get.User(url, config, db.Conn(config.Database)))
 
         if config.Pandas_au:

twint/user.py

@@ -9,7 +9,7 @@ class user:
 def inf(ur, _type):
     logme.debug(__name__+':inf')
     try:
-        group = ur.find("div", "user-actions btn-group not-following ")
+        group = ur.find("div", "profile")
         if group == None:
             group = ur.find("div", "user-actions btn-group not-following")
         if group == None:
@@ -18,14 +18,17 @@ def inf(ur, _type):
         print("Error: " + str(e))
 
     if _type == "id":
-        ret = group["data-user-id"]
+        screen_name = group.find("span", "screen-name").text
+        ret = ur.find("a", {"data-screenname": screen_name})
+        ret = ret.get('data-mentioned-user-id') if ret is not None else None
+        ret = "" if ret is None else ret
     elif _type == "name":
-        ret = group["data-name"]
+        ret = group.find("div", "fullname").text.split('\n')[0]
     elif _type == "username":
-        ret = group["data-screen-name"]
+        ret = group.find("span", "screen-name").text
     elif _type == "private":
-        ret = group["data-protected"]
-        if ret == 'true':
+        ret = group.find("div","protected")
+        if ret:
             ret = 1
         else:
             ret = 0
@@ -36,27 +39,29 @@ def card(ur, _type):
     logme.debug(__name__+':card')
     if _type == "bio":
         try:
-            ret = ur.find("p", "ProfileHeaderCard-bio u-dir").text.replace("\n", " ")
+            ret = ur.find("div", "bio").text.replace("\n", " ").strip()
         except:
             ret = ""
     elif _type == "location":
         try:
-            ret = ur.find("span", "ProfileHeaderCard-locationText u-dir").text
-            ret = ret[15:].replace("\n", " ")[:-10]
+            ret = ur.find("div", "location").text
         except:
             ret = ""
     elif _type == "url":
         try:
-            ret = ur.find("span", "ProfileHeaderCard-urlText u-dir").find("a")["title"]
+            ret = ur.find("link")["href"]
         except:
             ret = ""
 
     return ret
 
 def join(ur):
-    logme.debug(__name__+':join')
-    jd = ur.find("span", "ProfileHeaderCard-joinDateText js-tooltip u-dir")["title"]
-    return jd.split(" - ")
+    try:
+        logme.debug(__name__+':join')
+        jd = ur.find("span", "ProfileHeaderCard-joinDateText js-tooltip u-dir")["title"]
+        return jd.split(" - ")
+    except:
+        return ["", ""]
 
 def convertToInt(x):
     logme.debug(__name__+':contertToInt')
@@ -84,10 +89,13 @@ def convertToInt(x):
 
 def stat(ur, _type):
     logme.debug(__name__+':stat')
-    _class = f"ProfileNav-item ProfileNav-item--{_type}"
-    stat = ur.find("li", _class)
+    stats = ur.find('table', 'profile-stats')
+    stat_dict = {}
+    for stat in stats.find_all('td', 'stat'):
+        statnum, statlabel = stat.text.replace('\n', '').replace(',', '').split(' ')[:2]
+        stat_dict[statlabel.lower()] = int(statnum.replace(',', ''))
     try :
-        return int(stat.find("span", "ProfileNav-value")["data-count"])
+        return stat_dict[_type]
     except AttributeError:
         return 0
 
@@ -102,8 +110,8 @@ def media(ur):
 def verified(ur):
     logme.debug(__name__+':verified')
     try:
-        is_verified = ur.find("span", "ProfileHeaderCard-badges").text
-        if "Verified account" in is_verified:
+        is_verified = ur.find("img", {"alt": "Verified Account"})['alt']
+        if "Verified Account" in is_verified:
             is_verified = 1
         else:
             is_verified = 0
@@ -125,13 +133,13 @@ def User(ur):
     u.url = card(ur, "url")
     u.join_date = join(ur)[1]
     u.join_time = join(ur)[0]
-    u.tweets = stat(ur, "tweets is-active")
+    u.tweets = stat(ur, "tweets")
     u.following = stat(ur, "following")
     u.followers = stat(ur, "followers")
-    u.likes = stat(ur, "favorites")
-    u.media_count = media(ur)
+    u.likes = ""  # stat(ur, "favorites")
+    u.media_count = ""  # media(ur)
     u.is_private = inf(ur, "private")
     u.is_verified = verified(ur)
-    u.avatar = ur.find("img", "ProfileAvatar-image")["src"]
-    u.background_image = ur.find('div',{'class':'ProfileCanopy-headerBg'}).find('img').get('src')
+    u.avatar = ur.find("img", {"alt": u.name})["src"]
+    #u.background_image = ur.find('div',{'class':'ProfileCanopy-headerBg'}).find('img').get('src')
     return u