Skip to content

G
gfwlist
Commit 8c041c13 authored by Christopher Meng's avatar Christopher Meng
Browse files
Options

Move scripts to https://github.com/gfwlist/apollyon 

Add LGPLv2 LICENSE
parent 6972e43e
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 502 additions and 377 deletions
COPYING.txt 0 → 100644
View file @ 8c041c13
This diff is collapsed.
addChecksum.pl deleted 100755 → 0
View file @ 6972e43e
#!/usr/bin/perl
#############################################################################
# This is a reference script to add checksums to downloadable #
# subscriptions. The checksum will be validated by AutoProxy on download #
# and checksum mismatches (broken downloads) will be rejected. #
# #
# To add a checksum to a subscription file, run the script like this: #
# #
# perl addChecksum.pl subscription.txt #
# #
# Note: your subscription file should be saved in UTF-8 encoding, otherwise #
# the generated checksum might be incorrect. #
# #
#############################################################################
use strict;
use warnings;
use Digest::MD5 qw(md5_base64);
use File::stat;
use POSIX qw(locale_h);
use POSIX qw(strftime);
die "Usage: $^X $0 subscription.txt\n" unless @ARGV;
my $file = $ARGV[0];
my $data = readFile($file);
# Remove already existing checksum
$data =~ s/^.*!\s*checksum[\s\-:]+([\w\+\/=]+).*\n//gmi;
# Update timestamp
setlocale(LC_TIME, "C");
my $timestamp = strftime("%a, %d %b %Y %H:%M:%S %z", localtime(stat($file)->mtime));
$data =~ s/^!\s*Last Modified:.*$/! Last Modified: $timestamp/mi;
# Calculate new checksum: remove all CR symbols and empty
# lines and get an MD5 checksum of the result (base64-encoded,
# without the trailing = characters).
my $checksumData = $data;
$checksumData =~ s/\r//g;
$checksumData =~ s/\n+/\n/g;
# Calculate new checksum
my $checksum = md5_base64($checksumData);
# Insert checksum into the file
$data =~ s/(\r?\n)/$1! Checksum: $checksum$1/;
writeFile($file, $data);
sub readFile
{
my $file = shift;
open(local *FILE, "<", $file) || die "Could not read file '$file'";
binmode(FILE);
local $/;
my $result = <FILE>;
close(FILE);
return $result;
}
sub writeFile
{
my ($file, $contents) = @_;
open(local *FILE, ">", $file) || die "Could not write file '$file'";
binmode(FILE);
print FILE $contents;
close(FILE);
}
checkRules.py deleted 100755 → 0
View file @ 6972e43e
#!/usr/bin/python -u
# vim: ai et fileencoding=utf-8 ts=4 sw=4:
'''
This script will read `list.txt', and write both stdout and `list.err'.
Rules which are comments or regexs will be ignored.
* For `.example.com', if you got anything other than 56, then unless the
page contains DPI keyword, consider the rule invalid.
* For `||example.com', if you got anything other than 28, then consider
the rule invalid. (For issue 117, see below.)
* For `|https://*.example.com', if you got anything other than 35, check
it manually before considering it invalid.
In addition, if you got 6 or 7, check the rule again manually. If in
doubt, check the rule again manually. Also, please remember: `Garbage
in, garbage out.'
XXX: As a workaround for issue 117, `|http://example.com/' will be
tested as `.example.com/'.
'''
from urllib import unquote
import re
import subprocess
import sys
testurl = "http://%s/?"
iplist = ["66.249.89.19", "66.249.89.32", "66.249.89.33", "66.249.89.34", "66.249.89.35", "66.249.89.37", "66.249.89.38", "66.249.89.39", "66.249.89.40", "66.249.89.44", "66.249.89.45", "66.249.89.46", "66.249.89.47", "66.249.89.49", "66.249.89.50", "66.249.89.51", "66.249.89.52", "66.249.89.53", "66.249.89.54", "66.249.89.56", "66.249.89.57", "66.249.89.58", "66.249.89.59", "66.249.89.60", "66.249.89.61", "66.249.89.62", "66.249.89.63", "66.249.89.64", "66.249.89.65", "66.249.89.66", "66.249.89.68", "66.249.89.69", "66.249.89.72", "66.249.89.73", "66.249.89.74", "66.249.89.76", "66.249.89.77", "66.249.89.78", "66.249.89.79", "66.249.89.81", "66.249.89.82", "66.249.89.83", "66.249.89.84", "66.249.89.91", "66.249.89.93", "66.249.89.95", "66.249.89.96", "66.249.89.98", "66.249.89.99", "66.249.89.100", "66.249.89.101", "66.249.89.104", "66.249.89.112", "66.249.89.115", "66.249.89.116", "66.249.89.118", "66.249.89.120", "66.249.89.123", "66.249.89.128", "66.249.89.132", "66.249.89.136", "66.249.89.137", "66.249.89.141", "66.249.89.142", "66.249.89.143", "66.249.89.144", "66.249.89.145", "66.249.89.146", "66.249.89.148", "66.249.89.149", "66.249.89.152", "66.249.89.154", "66.249.89.155", "66.249.89.161", "66.249.89.162", "66.249.89.163", "66.249.89.164", "66.249.89.165", "66.249.89.184", "66.249.89.189", "66.249.89.190", "66.249.89.193", "66.249.89.210",
"66.102.13.17", "66.102.13.18", "66.102.13.19", "66.102.13.24", "66.102.13.32", "66.102.13.33", "66.102.13.34", "66.102.13.35", "66.102.13.36", "66.102.13.37", "66.102.13.38", "66.102.13.39", "66.102.13.40", "66.102.13.41", "66.102.13.42", "66.102.13.43", "66.102.13.44", "66.102.13.45", "66.102.13.46", "66.102.13.47", "66.102.13.48", "66.102.13.49", "66.102.13.51", "66.102.13.52", "66.102.13.53", "66.102.13.54", "66.102.13.56", "66.102.13.57", "66.102.13.58", "66.102.13.59", "66.102.13.60", "66.102.13.61", "66.102.13.62", "66.102.13.63", "66.102.13.64", "66.102.13.65", "66.102.13.66", "66.102.13.68", "66.102.13.69", "66.102.13.72", "66.102.13.73", "66.102.13.74", "66.102.13.75", "66.102.13.76", "66.102.13.77", "66.102.13.78", "66.102.13.79", "66.102.13.81", "66.102.13.82", "66.102.13.83", "66.102.13.84", "66.102.13.91", "66.102.13.93", "66.102.13.95", "66.102.13.96", "66.102.13.98", "66.102.13.99", "66.102.13.100", "66.102.13.101", "66.102.13.102", "66.102.13.103", "66.102.13.104", "66.102.13.105", "66.102.13.106", "66.102.13.112", "66.102.13.113", "66.102.13.115", "66.102.13.118", "66.102.13.120", "66.102.13.123", "66.102.13.128", "66.102.13.132", "66.102.13.136", "66.102.13.137", "66.102.13.138", "66.102.13.139", "66.102.13.141", "66.102.13.142", "66.102.13.143", "66.102.13.144", "66.102.13.145", "66.102.13.146", "66.102.13.147", "66.102.13.148", "66.102.13.149", "66.102.13.152", "66.102.13.154", "66.102.13.155", "66.102.13.156", "66.102.13.157", "66.102.13.161", "66.102.13.162", "66.102.13.163", "66.102.13.164", "66.102.13.165", "66.102.13.166", "66.102.13.167", "66.102.13.178", "66.102.13.184", "66.102.13.190", "66.102.13.191", "66.102.13.193", "66.102.13.210"]
offset = 0
INVALID = 0
IP = 1
TLS = 2
URL = 4
expect = {IP: 28, TLS: 35, URL: 56, INVALID: 0}
def getUrl(rule):
global offset
if rule.startswith('||'): return ('http://' + rule[2:], IP)
if rule.startswith('|https'): return (rule[1:], TLS)
if rule.startswith('|http://'):
rule = '.' + rule[8:] # XXX: issue 117
offset += 1
if offset >= len(iplist): offset = 0
return (testurl % iplist[offset] + rule, URL)
def main():
fin = open('list.txt', 'r')
ferr = open('list.err', 'w')
line = 0
if len(sys.argv) > 1:
startfrom = int(sys.argv[1])
else:
startfrom = 0
for rule in fin:
line += 1
rule = rule.strip()
if line < startfrom: continue
if not rule: continue
if rule.startswith('[AutoProxy'): continue
if rule.startswith('!'): continue
if rule.startswith('@@'): continue
if rule.startswith('/') and rule.endswith('/'): continue
(test, t) = getUrl(rule)
val1 = subprocess.call(['/usr/bin/curl', '-4', '-I', '-m', '5', test], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
val2 = subprocess.call(['/usr/bin/curl', '-4', '-I', '-m', '5', test], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if (t==IP and val1==28 and val2==28) or (t==TLS and val1==35 or val2==35) or (t==URL and val1==56 or val2==56):
pass
elif t==IP and val1==56 or val2==56:
print line, '\t', test, '=>', '\033[31mexpecting %d, got %d %d\033[0m' % (expect[t], val1, val2)
else:
print line, '\t', test, '=>', '\033[1;31mexpecting %d, got %d %d\033[0m' % (expect[t], val1, val2)
ferr.write(str(line) + ': "' + rule + '", expecting %d, got %d %d' % (expect[t], val1, val2) + '\n')
if __name__ == '__main__':
main()
lookup.sh deleted 100755 → 0
View file @ 6972e43e
#!/bin/bash
if [ $# != 1 ]; then
echo -e "\
Usage:
$./lookup.sh wordpress
$./lookup.sh ghs.l.google
$./lookup.sh 72.14.203.121 // ghs
$./lookup.sh 67.207.139.81 // Posterous
$./lookup.sh 72.32.231.8 // Tumblr
$./lookup.sh typepad
";
exit 1;
fi
while read line
do
host=$( echo "$line" |
grep -oE "[a-z0-9]([a-z0-9_\.\-]*[a-z0-9])?\.[a-z]{2,4}" |
grep -vE "(aspx?|dotn|exe|fan|html?|php|zh)$"
);
[ "$host" ] && [ "$(nslookup $host | grep -i $1)" ] && echo $host;
done < "list.txt"
sendGFWList.sh deleted 100755 → 0
View file @ 6972e43e
#!/bin/bash
#
# A simple script help to maintain AutoProxy gfwList easily.
#
# Features:
# Update local svn repository;
# Commit decoded changes(by others in your team) to local git repository
# with decoded message and authors name;
# Update "Last Modified" & "Checksum";
# Commit your changes to local git repository;
# Commit your encoded changes to remote svn server with encoded log;
# Plus some error handling.
# Usage:
# Initialize:
# $svn checkout https://autoproxy-gfwlist.googlecode.com/svn/trunk/ gfwList --username your-google-user-name
# $cd gfwList
# $git init
# $openssl base64 -d -in gfwlist.txt -out list.txt
# $git add list.txt
# $git commit -a -m "init"
# Normal Usage:
# edit list.txt as usual;
# $./sendGFWList.sh "say something about this edit"
# Note:
# 1: You can use "$git log" "$git show" "$git diff"...;
# 2: Do NOT commit "list.txt" to svn server (it won't by default);
# 3: Do NOT use any unicode character in the list, there is a known bug;
# 4: Do NOT "svn update", run this script to update / commit at any time.
################################################################################
# dependence
for cmd in sed openssl awk svn git perl file
do
which $cmd &> /dev/null;
if [ $? -ne 0 ]; then
echo "Error: depends on $cmd, please install it first.";
exit 1;
fi
done
# get formated author and log information
log=$(svn log --xml -r BASE:HEAD) || exit 1;
log=$(echo $log | awk -v RS='' -F '</?author>|</?msg>' '{ for(i=6;i<NF;i+=4) printf "%s:%s;",$i,$(i+2); }') &&
# convert from base64
i=0 &&
convertedLog="" &&
while [ "$log" != "" ]
do
if (( $i%2 == 0 )); then # author
temp=${log%%:*};
convertedLog+=${temp%@*}; # don't include "@gmail.com"
convertedLog+=": ";
# discard used string
log=${log#*:};
else # log, decode it
temp=$( echo ${log%%;*} | openssl base64 -d );
convertedLog+=$temp;
convertedLog+="\n";
log=${log#*;};
fi
((i++));
done
# modified by others, commit to local git repository
if [ "$convertedLog" != "" ]; then
git diff > temp.patch &&
svn update || exit 1;
openssl base64 -d -in gfwlist.txt -out list.txt &&
./validateChecksum.pl list.txt;
if [ $? -ne 0 ]; then
# recover, discard broken list.txt
git checkout list.txt && git apply temp.patch && rm temp.patch;
echo -e "\n\n\n*********************************************************\n";
echo "Error: gfwlist.txt from svn is invalid!!!";
echo "It must be a download error or somebody made a mistake.";
echo -e "\nYou can simply run this script again to fix the problem.";
echo "But wait...!"
echo "This would overwrite all commits till your last update!!!";
echo -e "\nIf you are confused, wait somebody else to fix it.";
echo "Please always report this to our maintainers' group!";
echo -e "\n*********************************************************\n\n\n";
exit 1;
fi
echo -e $convertedLog | git commit -a -F - &&
[ -s temp.patch ] && git apply temp.patch &&
rm temp.patch;
if [ -s temp.patch ]; then
echo -e "\n\033[31mError:\033[0m git apply failed, your work saved at temp.patch\n";
exit 1;
elif [ -a temp.patch ]; then
# empty, remove it
rm temp.patch;
fi
fi
if [ "$(git diff)" == "" ]; then
echo "Info: list.txt not modified.";
exit 0;
fi
if [ "$*" == "" ]; then
echo "Error: empty log, please say something about this modification.";
exit 1;
fi
# update date and checksum
./addChecksum.pl list.txt &&
if [ "$(file -b list.txt)" != "ASCII text" ]; then
echo "Error: list.txt invalid, please make sure:";
echo "1. there is no non-ASCII characters;";
echo "2. configure your text editor to use unix style line break.";
exit 1;
fi
# save local changes to git & svn
# if conflict or network problem occurs: do nothing & throw error message
git commit -a -m "$*" &&
(
openssl base64 -in list.txt |
# convert dos new line to unix style, old mac style ignored
tr -d '\r' > gfwlist.txt &&
# may be failed because of connection/authentication problems
svn ci gfwlist.txt -m $( echo "$*" | openssl base64 | tr -d '\r\n' ) ||
# "svn ci" and "git commit" are atomic operations
git reset HEAD^ 1> /dev/null;
) &&
# BASE++ if committed
svn update 1> /dev/null;
validateChecksum.pl deleted 100755 → 0
View file @ 6972e43e
#!/usr/bin/perl
#############################################################################
# This is a reference script to validate the checksum in downloadable #
# subscription. This performs the same validation as Adblock Plus when it #
# downloads the subscription. #
# #
# To validate a subscription file, run the script like this: #
# #
# perl validateChecksum.pl subscription.txt #
# #
# Note: your subscription file should be saved in UTF-8 encoding, otherwise #
# the validation result might be incorrect. #
# #
# 20100418: Stolen from ABP with minor modification for AutoProxy project #
#############################################################################
use strict;
use warnings;
use Digest::MD5 qw(md5_base64);
die "Usage: $^X $0 subscription.txt\n" unless @ARGV;
my $file = $ARGV[0];
my $data = readFile($file);
# Normalize data
$data =~ s/\r//g;
$data =~ s/\n+/\n/g;
# Extract checksum
# Remove checksum
$data =~ s/^\s*!\s*checksum[\s\-:]+([\w\+\/=]+).*\n//mi;
my $checksum = $1;
die "Error: couldn't find a checksum in the file\n" unless $checksum;
# Calculate new checksum
my $checksumExpected = md5_base64($data);
# Compare checksums
die "Error: invalid checksum\n" unless $checksum eq $checksumExpected;
sub readFile
{
my $file = shift;
open(local *FILE, "<", $file) || die "Error: could not read file '$file'";
binmode(FILE);
local $/;
my $result = <FILE>;
close(FILE);
return $result;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment