Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
T
tx3-bang-reader
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Locked Files
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Security & Compliance
Security & Compliance
Dependency List
License Compliance
Packages
Packages
List
Container Registry
Analytics
Analytics
CI / CD
Code Review
Insights
Issues
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nanahira
tx3-bang-reader
Commits
7241e206
Commit
7241e206
authored
Jul 01, 2020
by
nanahira
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
finish
parent
d6864893
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
144 additions
and
21 deletions
+144
-21
package.json
package.json
+1
-1
src/fetcher.ts
src/fetcher.ts
+111
-16
src/proxy.ts
src/proxy.ts
+13
-4
src/run.ts
src/run.ts
+19
-0
No files found.
package.json
View file @
7241e206
...
@@ -5,7 +5,7 @@
...
@@ -5,7 +5,7 @@
"main"
:
"build/run.js"
,
"main"
:
"build/run.js"
,
"scripts"
:
{
"scripts"
:
{
"build"
:
"./node_modules/.bin/tsc"
,
"build"
:
"./node_modules/.bin/tsc"
,
"
test"
:
"node build/tests/test
.js"
"
fetch"
:
"node build/src/run
.js"
},
},
"repository"
:
{
"repository"
:
{
"type"
:
"git"
,
"type"
:
"git"
,
...
...
src/fetcher.ts
View file @
7241e206
import
axios
from
"
axios
"
;
import
axios
from
"
axios
"
;
import
_
from
"
underscore
"
;
import
_
from
"
underscore
"
;
import
parseHTML
from
"
posthtml-parser
"
;
import
{
User
}
from
"
./user
"
;
import
{
User
}
from
"
./user
"
;
import
{
ProxyFetcher
}
from
"
./proxy
"
;
import
{
PlayerRow
,
parsePlayerRows
}
from
"
./playerlist
"
;
import
qs
from
"
querystring
"
;
class
Tx3Fetcher
{
export
const
servers
=
[
async
fetchRole
(
id
:
string
):
Promise
<
User
>
{
"
东方明珠
"
,
//try {
"
君临天下
"
,
const
content
=
(
await
axios
.
get
(
`http://bang.tx3.163.com/bang/role/
${
id
}
`
,
{
"
幻龙诀
"
,
"
气壮山河
"
,
"
紫禁之巅
"
,
"
绝代风华
"
,
"
问鼎天下
"
,
"
鼎立山河
"
,
"
天下无双
"
,
"
情动大荒
"
,
"
昆仑变
"
,
"
琉璃月
"
,
"
齐鲁天下
"
,
"
剑舞香江
"
,
"
白云山
"
,
"
瘦西湖
"
,
"
逐鹿中原
"
,
"
天府之国
"
,
"
黄鹤楼
"
,
"
墨倾天下
"
,
"
武夷九曲
"
,
"
上善若水
"
,
"
飞龙在天
"
,
"
烽火关东
"
,
"
盛世长安
"
,
"
荒火山神
"
,
"
一诺山海
"
,
"
梦回山海
"
,
"
锦绣花朝
"
]
export
class
Tx3Fetcher
{
proxyFetcher
:
ProxyFetcher
;
constructor
()
{
this
.
proxyFetcher
=
new
ProxyFetcher
();
}
async
initProxies
()
{
await
this
.
proxyFetcher
.
initProxies
();
}
async
fetch
():
Promise
<
Map
<
string
,
User
[]
>>
{
const
res
=
new
Map
<
string
,
User
[]
>
();
const
userLists
=
await
Promise
.
all
(
servers
.
map
(
server
=>
this
.
fetchUsersFromServer
(
server
)));
for
(
let
i
=
1
;
i
<
userLists
.
length
;
++
i
){
res
.
set
(
servers
[
i
],
userLists
[
i
])
}
return
res
;
}
async
fetchUsersFromServer
(
server
:
string
):
Promise
<
User
[]
>
{
const
userList
=
await
this
.
fetchListFromServer
(
server
);
return
await
Promise
.
all
(
userList
.
map
(
r
=>
this
.
fetchUser
(
r
)));
}
async
fetchListFromServer
(
server
:
string
):
Promise
<
PlayerRow
[]
>
{
console
.
log
(
`Fetching user list from server
${
server
}
.`
);
const
resPromises
:
Promise
<
PlayerRow
[]
>
[]
=
[];
for
(
let
school
=
1
;
school
<
12
;
++
school
)
{
resPromises
.
push
(
this
.
fetchListFromSchoolAndServer
(
school
,
server
));
}
const
result
=
_
.
flatten
(
await
Promise
.
all
(
resPromises
));
console
.
log
(
`Fetched user list with
${
result
.
length
}
users from server
${
server
}
.`
);
return
result
;
}
async
fetchListFromSchoolAndServer
(
school
:
number
,
server
:
string
):
Promise
<
PlayerRow
[]
>
{
console
.
log
(
`Fetching user list from server
${
server
}
with school
${
school
}
.`
);
const
res
:
PlayerRow
[][]
=
[];
for
(
let
page
=
1
;
page
<=
25
;
++
page
)
{
const
list
=
await
this
.
fetchUserList
(
school
,
server
,
page
);
if
(
!
list
.
length
)
{
console
.
log
(
`User list from server
${
server
}
with school
${
school
}
page
${
page
}
is blank.`
);
break
;
}
res
.
push
(
list
);
}
return
_
.
flatten
(
res
);
}
async
fetchUserList
(
school
:
number
,
server
:
string
,
page
:
number
):
Promise
<
PlayerRow
[]
>
{
console
.
log
(
`Fetching user list from server
${
server
}
with school
${
school
}
page
${
page
}
.`
);
try
{
const
content
:
string
=
await
this
.
proxyFetcher
.
getWithProxy
(
`http://bang.tx3.163.com/bang/ranks`
,
{
responseType
:
"
document
"
,
params
:
{
order_key
:
"
equ_xiuwei
"
,
count
:
20
,
school
,
server
,
page
},
paramsSerializer
:
(
params
)
=>
{
return
qs
.
stringify
(
params
);
}
});
return
parsePlayerRows
(
content
);
}
catch
(
e
)
{
console
.
error
(
`Errored fetching user list with params
${
school
}
${
server
}
${
page
}
}:
${
e
.
toString
()}
`
);
return
[];
}
}
async
fetchUser
(
playerRow
:
PlayerRow
):
Promise
<
User
>
{
const
id
=
playerRow
.
url
.
split
(
"
/
"
).
pop
();
try
{
console
.
log
(
`Fetching user
${
playerRow
.
name
}
from
${
playerRow
.
server
}
.`
);
const
content
:
string
=
await
this
.
proxyFetcher
.
getWithProxy
(
`http://bang.tx3.163.com
${
playerRow
.
url
}
`
,
{
responseType
:
"
document
"
responseType
:
"
document
"
})).
data
;
});
const
parsedContent
=
parseHTML
(
content
);
const
user
=
new
User
(
id
,
content
,
playerRow
.
region
);
return
new
User
(
id
,
content
,
null
);
console
.
log
(
`Fetched user
${
playerRow
.
name
}
from
${
playerRow
.
server
}
.`
);
/*} catch(e) {
return
user
;
}
catch
(
e
)
{
console
.
error
(
`Errored fetching role data from
${
id
}
:
${
e
.
toString
()}
`
);
console
.
error
(
`Errored fetching role data from
${
id
}
:
${
e
.
toString
()}
`
);
return
null
;
return
null
;
}
*/
}
}
}
}
}
async
function
main
()
{
const
fetcher
=
new
Tx3Fetcher
();
console
.
log
(
JSON
.
stringify
(
await
fetcher
.
fetchRole
(
"
28_20588
"
),
null
,
2
));
}
main
();
src/proxy.ts
View file @
7241e206
...
@@ -8,6 +8,9 @@ export class ProxyFetcher {
...
@@ -8,6 +8,9 @@ export class ProxyFetcher {
this
.
counter
=
0
;
this
.
counter
=
0
;
}
}
async
initProxiesFrom
(
url
:
string
)
{
async
initProxiesFrom
(
url
:
string
)
{
if
(
process
.
env
.
NO_PROXY
)
{
return
;
}
while
(
true
)
{
while
(
true
)
{
try
{
try
{
const
proxyPage
:
string
=
(
await
axios
.
get
(
url
,
{
const
proxyPage
:
string
=
(
await
axios
.
get
(
url
,
{
...
@@ -33,19 +36,25 @@ export class ProxyFetcher {
...
@@ -33,19 +36,25 @@ export class ProxyFetcher {
}
}
async
getWithProxy
(
url
:
string
,
options
:
AxiosRequestConfig
)
{
async
getWithProxy
(
url
:
string
,
options
:
AxiosRequestConfig
)
{
while
(
true
)
{
while
(
true
)
{
if
(
!
this
.
proxies
.
length
)
{
if
(
!
process
.
env
.
NO_PROXY
&&
!
this
.
proxies
.
length
)
{
await
this
.
initProxies
();
await
this
.
initProxies
();
}
}
const
proxyIndex
=
(
++
this
.
counter
)
%
this
.
proxies
.
length
;
const
proxyIndex
=
process
.
env
.
NO_PROXY
?
null
:
(
++
this
.
counter
)
%
this
.
proxies
.
length
;
const
proxy
=
this
.
proxies
[
proxyIndex
];
const
proxy
=
process
.
env
.
NO_PROXY
?
null
:
this
.
proxies
[
proxyIndex
];
try
{
try
{
const
data
=
(
await
axios
.
get
(
url
,
{
const
data
=
(
await
axios
.
get
(
url
,
{
proxy
,
proxy
,
headers
:
{
"
User-Agent
"
:
"
Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:77.0) Gecko/20100101 Firefox/77.0
"
},
timeout
:
5000
,
...
options
...
options
})).
data
;
})).
data
;
return
data
;
return
data
;
}
catch
(
e
)
{
}
catch
(
e
)
{
this
.
proxies
.
splice
(
proxyIndex
,
1
);
if
(
!
process
.
env
.
NO_PROXY
)
{
this
.
proxies
.
splice
(
proxyIndex
,
1
);
}
console
.
error
(
`Failed fetching data from
${
url
}
:
${
e
.
toString
()}
`
)
console
.
error
(
`Failed fetching data from
${
url
}
:
${
e
.
toString
()}
`
)
}
}
}
}
...
...
src/run.ts
0 → 100644
View file @
7241e206
import
{
Tx3Fetcher
,
servers
}
from
"
./fetcher
"
;
import
fs
from
"
fs
"
;
import
_
from
"
underscore
"
;
async
function
main
()
{
try
{
await
fs
.
promises
.
access
(
"
./output/servers
"
);
}
catch
(
e
)
{
await
fs
.
promises
.
mkdir
(
"
./output/servers
"
,
{
recursive
:
true
});
}
const
fetcher
=
new
Tx3Fetcher
();
await
fetcher
.
initProxies
();
const
fetchResult
=
await
fetcher
.
fetch
();
const
allUsers
=
_
.
flatten
(
Array
.
from
(
fetchResult
.
values
));
await
fs
.
promises
.
writeFile
(
"
./output/all.json
"
,
JSON
.
stringify
(
allUsers
,
null
,
2
));
await
Promise
.
all
(
servers
.
map
(
server
=>
fs
.
promises
.
writeFile
(
`./output/servers/
${
server
}
.json`
,
JSON
.
stringify
(
fetchResult
.
get
(
server
),
null
,
2
))));
}
main
();
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment