Browse Source

Changed backend from requests to wget.

master
Alec Murphy 5 years ago
parent
commit
8ea25198d3
  1. 4
      uriel_preprocessor.py
  2. 9
      uriel_proxy

4
uriel_preprocessor.py

@ -6,7 +6,7 @@ def preprocess(htm1):
hb_header = '$WW,1$$BLACK$$MA+LIS,"[Close]",LM="CloseBrowser;"$ $MA+LIS,"[Back]",LM="Browser(\\"h:back\\");"$ $MA+LIS,"[Fwd]",LM="Browser(\\"h:fwd\\");"$ $MA+LIS,"[Go]",LM="Browser(GetStr(\\"\nURL> \\"));"$ ' + title_text + '\n\n'
if htm1.upper().find('<HTML') == -1:
return re.sub(r'[^\x00-\x7F]','', hb_header) + re.sub(r'[^\x00-\x7F]','', htm1)
return hb_header + htm1
htm1 = htm1[htm1.upper().find('<HTML'):]
htm1 = htm1.replace('$', '$$')
@ -144,4 +144,4 @@ def preprocess(htm1):
hb_header = '$WW,1$$BLACK$$MA+LIS,"[Close]",LM="CloseBrowser;"$ $MA+LIS,"[Back]",LM="Browser(\\"h:back\\");"$ $MA+LIS,"[Fwd]",LM="Browser(\\"h:fwd\\");"$ $MA+LIS,"[Go]",LM="Browser(GetStr(\\"URL> \\"));"$ ' + title_text + '\n\n'
return re.sub(r'[^\x00-\x7F]','', hb_header) + re.sub(r'[^\x00-\x7F]','', html)
return hb_header + html

9
uriel_proxy

@ -1,7 +1,6 @@
#!/usr/bin/python
from uriel_preprocessor import preprocess
import os, socket, sys, time, urlparse
import requests
import os, socket, subprocess, sys, time, urlparse
HOST = '127.0.0.1'
PORT = 7202
@ -52,8 +51,7 @@ while 1:
blk_ctr = 0
if get_file.find('://') != -1:
headers = { 'User-Agent': URIEL_VER_STR + ' (' + cmd_in.split('^')[0].rsplit('|')[1] + ')' }
r = requests.get(get_file, headers=headers)
file = r.text
file = subprocess.Popen('wget -O - -U "' + headers['User-Agent'] + '" "' + get_file + '" 2>/dev/null', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0]
else:
file = open(get_file, "rb").read()
while blk_ctr < len(file):
@ -110,8 +108,7 @@ while 1:
url = rel_url + url
url = url[:url.find('//')] + '//' + url[url.find('//')+2:].replace('//','/')
headers = { 'User-Agent': user_agent }
r = requests.get(url, headers=headers)
data = r.text
data = subprocess.Popen('wget -O - -U "' + headers['User-Agent'] + '" "' + url + '" 2>/dev/null', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0]
page = preprocess(data)
hst_index += 1
history = history[0:hst_index]

Loading…
Cancel
Save