|
|
|
@ -1,7 +1,6 @@
|
|
|
|
|
#!/usr/bin/python |
|
|
|
|
from uriel_preprocessor import preprocess |
|
|
|
|
import os, socket, sys, time, urlparse |
|
|
|
|
import requests |
|
|
|
|
import os, socket, subprocess, sys, time, urlparse |
|
|
|
|
|
|
|
|
|
HOST = '127.0.0.1' |
|
|
|
|
PORT = 7202 |
|
|
|
@ -52,8 +51,7 @@ while 1:
|
|
|
|
|
blk_ctr = 0 |
|
|
|
|
if get_file.find('://') != -1: |
|
|
|
|
headers = { 'User-Agent': URIEL_VER_STR + ' (' + cmd_in.split('^')[0].rsplit('|')[1] + ')' } |
|
|
|
|
r = requests.get(get_file, headers=headers) |
|
|
|
|
file = r.text |
|
|
|
|
file = subprocess.Popen('wget -O - -U "' + headers['User-Agent'] + '" "' + get_file + '" 2>/dev/null', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0] |
|
|
|
|
else: |
|
|
|
|
file = open(get_file, "rb").read() |
|
|
|
|
while blk_ctr < len(file): |
|
|
|
@ -110,8 +108,7 @@ while 1:
|
|
|
|
|
url = rel_url + url |
|
|
|
|
url = url[:url.find('//')] + '//' + url[url.find('//')+2:].replace('//','/') |
|
|
|
|
headers = { 'User-Agent': user_agent } |
|
|
|
|
r = requests.get(url, headers=headers) |
|
|
|
|
data = r.text |
|
|
|
|
data = subprocess.Popen('wget -O - -U "' + headers['User-Agent'] + '" "' + url + '" 2>/dev/null', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0] |
|
|
|
|
page = preprocess(data) |
|
|
|
|
hst_index += 1 |
|
|
|
|
history = history[0:hst_index] |
|
|
|
|