Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
T
tx3-bang-reader
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Locked Files
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Security & Compliance
Security & Compliance
Dependency List
License Compliance
Packages
Packages
List
Container Registry
Analytics
Analytics
CI / CD
Code Review
Insights
Issues
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nanahira
tx3-bang-reader
Commits
cfb001c8
Commit
cfb001c8
authored
Jul 01, 2020
by
nanahira
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
fetch list only
parent
8baa9789
Pipeline
#215
passed with stages
in 1 minute and 12 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
36 additions
and
8 deletions
+36
-8
src/fetcher.ts
src/fetcher.ts
+27
-1
src/proxy.ts
src/proxy.ts
+8
-6
src/run.ts
src/run.ts
+1
-1
No files found.
src/fetcher.ts
View file @
cfb001c8
...
...
@@ -76,6 +76,32 @@ export class Tx3Fetcher {
return
_
.
flatten
(
res
);
}
async
fetchUsers
(
school
:
number
,
server
:
string
,
page
:
number
):
Promise
<
User
[]
>
{
const
playerRows
=
await
this
.
fetchList
(
school
,
server
,
page
);
return
await
Promise
.
all
(
playerRows
.
map
(
row
=>
this
.
fetchUser
(
row
)));
}
async
fetchListFromServer
(
server
:
string
):
Promise
<
PlayerRow
[]
>
{
console
.
log
(
`Fetching user list from server
${
server
}
.`
);
const
resPromises
:
Promise
<
PlayerRow
[]
>
[]
=
[];
for
(
let
school
=
1
;
school
<
12
;
++
school
)
{
resPromises
.
push
(
this
.
fetchListFromSchoolAndServer
(
school
,
server
));
}
const
result
=
_
.
flatten
(
await
Promise
.
all
(
resPromises
));
console
.
log
(
`Fetched user list with
${
result
.
length
}
users from server
${
server
}
.`
);
return
result
;
}
async
fetchListFromSchoolAndServer
(
school
:
number
,
server
:
string
):
Promise
<
PlayerRow
[]
>
{
console
.
log
(
`Fetching users from server
${
server
}
with school
${
school
}
.`
);
const
res
:
PlayerRow
[][]
=
[];
for
(
let
page
=
1
;
page
<=
25
;
++
page
)
{
const
list
=
await
this
.
fetchList
(
school
,
server
,
page
);
if
(
!
list
.
length
)
{
break
;
}
res
.
push
(
list
);
}
return
_
.
flatten
(
res
);
}
async
fetchList
(
school
:
number
,
server
:
string
,
page
:
number
):
Promise
<
PlayerRow
[]
>
{
console
.
log
(
`Fetching user list from server
${
server
}
with school
${
school
}
page
${
page
}
.`
);
try
{
const
content
:
string
=
await
this
.
proxyFetcher
.
getWithProxy
(
`http://bang.tx3.163.com/bang/ranks`
,
{
...
...
@@ -92,7 +118,7 @@ export class Tx3Fetcher {
}
});
const
playerRows
=
parsePlayerRows
(
content
);
return
await
Promise
.
all
(
playerRows
.
map
(
row
=>
this
.
fetchUser
(
row
)))
;
return
playerRows
;
}
catch
(
e
)
{
console
.
error
(
`Errored fetching user list with params
${
school
}
${
server
}
${
page
}
}:
${
e
.
toString
()}
`
);
return
[];
...
...
src/proxy.ts
View file @
cfb001c8
import
axios
,
{
AxiosProxyConfig
,
AxiosRequestConfig
}
from
"
axios
"
;
const
NO_PROXY
=
true
;
const
proxySourceList
=
[
"
http://www.89ip.cn/tqdl.html?api=1&num=9999
"
,
"
http://www.66ip.cn/mo.php?tqsl=9999
"
]
...
...
@@ -70,7 +72,7 @@ export class ProxyFetcher {
this
.
counter
=
0
;
}
async
initProxiesFrom
(
url
:
string
)
{
if
(
process
.
env
.
NO_PROXY
)
{
if
(
NO_PROXY
)
{
return
;
}
console
.
log
(
`Fetching proxies from
${
url
}
.`
)
...
...
@@ -103,24 +105,24 @@ export class ProxyFetcher {
}
async
getWithProxy
(
url
:
string
,
options
:
AxiosRequestConfig
)
{
while
(
true
)
{
if
(
!
process
.
env
.
NO_PROXY
&&
!
this
.
proxies
.
length
)
{
if
(
!
NO_PROXY
&&
!
this
.
proxies
.
length
)
{
await
this
.
initProxies
();
}
const
proxyIndex
=
process
.
env
.
NO_PROXY
?
null
:
(
++
this
.
counter
)
%
this
.
proxies
.
length
;
const
proxyIndex
=
NO_PROXY
?
null
:
(
++
this
.
counter
)
%
this
.
proxies
.
length
;
//const proxyIndex = 0;
const
proxy
=
process
.
env
.
NO_PROXY
?
null
:
this
.
proxies
[
proxyIndex
];
const
proxy
=
NO_PROXY
?
null
:
this
.
proxies
[
proxyIndex
];
try
{
const
data
=
(
await
axios
.
get
(
url
,
{
proxy
,
headers
:
{
"
User-Agent
"
:
agentList
[
this
.
counter
%
agentList
.
length
]
},
timeout
:
5
000
,
timeout
:
30
000
,
...
options
})).
data
;
return
data
;
}
catch
(
e
)
{
if
(
!
process
.
env
.
NO_PROXY
)
{
if
(
!
NO_PROXY
)
{
this
.
proxies
.
splice
(
proxyIndex
,
1
);
}
console
.
error
(
`Failed fetching data from
${
url
}
:
${
e
.
toString
()}
${
this
.
proxies
.
length
}
proxies left.`
)
...
...
src/run.ts
View file @
cfb001c8
...
...
@@ -5,7 +5,7 @@ import _ from "underscore";
const
fetcher
=
new
Tx3Fetcher
();
async
function
runServer
(
server
:
string
)
{
const
users
=
await
fetcher
.
fetch
Users
FromServer
(
server
);
const
users
=
await
fetcher
.
fetch
List
FromServer
(
server
);
await
fs
.
promises
.
writeFile
(
`./output/servers/
${
server
}
.json`
,
JSON
.
stringify
(
users
,
null
,
2
));
return
users
;
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment