Browse Source

Inline images support, file downloads

master
Alec Murphy 5 years ago
parent
commit
4a9c6d00fb
  1. 41
      Uriel.HC
  2. 77
      uriel_preprocessor.py
  3. 34
      uriel_proxy

41
Uriel.HC

@ -2,19 +2,26 @@
#define URIEL_BINARY_SOF "*[U_SOF]"
#define URIEL_FCMD_GET "*[U_GET]"
#define URIEL_FCMD_PUT "*[U_PUT]"
#define URIEL_TMP_IMG "/Tmp/uriel_img.bmp"
#define URIEL_PORT 2
#define URIEL_RAW_PORT 0x02F8
#define URIEL_TIMEOUT 1
#include "::/Doc/Comm"
U8 UrielPageBuf[2097152];
U8 UrielFileBuf[10485760];
I64 UrielFileBufSize=0;
I64 UrielPageBufSize=0;
I64 UrielGetFileSuccess=0;
CComm *c=CommInit8n1(URIEL_PORT,115200);
U0 Get(U8 *remote_file, U8 *local_file="file") {
UrielGetFileSuccess=0;
Bool load=FALSE;
I64 b=0;
I64 PrevFileBufSize=0;
I64 StartTimer=0;
I64 TickTimer=0;
UrielFileBufSize=0;
CommPutS(URIEL_PORT, URIEL_FCMD_GET);
CommPutS(URIEL_PORT, remote_file);
@ -25,7 +32,9 @@ U0 Get(U8 *remote_file, U8 *local_file="file") {
progress1=0;
StrCpy(progress1_desc, "Receiving file");
while (!load) {
if ((TickTimer-StartTimer)>URIEL_TIMEOUT) { UrielFileBufSize=0; break; };
if (InU8(0x60)==0x01) { UrielFileBufSize=0; break; };
PrevFileBufSize=UrielFileBufSize;
Sleep(0);
while (FifoU8Rem(c->RX_fifo,&b)) {
if (b != 0x00) {
@ -38,6 +47,7 @@ U0 Get(U8 *remote_file, U8 *local_file="file") {
if (StrCmp(UrielFileBuf+(UrielFileBufSize-8), URIEL_BINARY_EOF)==0) {
UrielFileBufSize -= 8;
load=TRUE;
UrielGetFileSuccess=1;
break;
}
else {
@ -48,6 +58,10 @@ U0 Get(U8 *remote_file, U8 *local_file="file") {
}
}
}
if (PrevFileBufSize==UrielFileBufSize) {
if (StartTimer==0) { StartTimer=SysTimerRead/1000000; };
TickTimer=SysTimerRead/1000000;
}
}
FileWrite(local_file, UrielFileBuf, UrielFileBufSize);
ProgressBarsRst;
@ -77,6 +91,28 @@ U0 Send(U8 *local_file, U8 *remote_file="") {
Free(d_file);
}
U0 InsertImg(U8 *img_anchor, U8 *img_sid, U8 *img_eid, U8 *img_url) {
UrielGetFileSuccess=0;
DocAnchorFind(DocPut, img_anchor);
Get(img_url, URIEL_TMP_IMG);
while (UrielGetFileSuccess==0) {
DocAnchorFind(DocPut, img_anchor);
Get("retry:send", URIEL_TMP_IMG);
}
DocAnchorFind(DocPut, img_anchor);
DocBMP(,URIEL_TMP_IMG);
if (StrCmp(img_sid, "")!=0) {
DocAnchorFind(DocPut, img_sid);
DocPrintPartial(DocPut, "$ID,14$");
}
if (StrCmp(img_eid, "")!=0) {
DocAnchorFind(DocPut, img_eid);
DocPrintPartial(DocPut, "$ID,-14$");
}
DocAnchorFind(DocPut, img_anchor);
UrielGetFileSuccess=0;
}
U0 CloseBrowser() {
Bool close=FALSE;
close = PopUpCancelOk("Close Browser?");
@ -137,3 +173,8 @@ U0 Browser(U8 *url)
Free(Cmd);
DocBottom;
}
U0 Navigate(U8 *anchor, U8 *url) {
DocAnchorFind(DocPut, anchor);
Browser(url);
}

77
uriel_preprocessor.py

@ -1,12 +1,15 @@
import bs4
import re, sys
def preprocess(htm1):
def preprocess(htm1, l_url):
title_text = ''
hb_header = '$WW,1$$BLACK$$MA+LIS,"[Close]",LM="CloseBrowser;"$ $MA+LIS,"[Back]",LM="Browser(\\"h:back\\");"$ $MA+LIS,"[Fwd]",LM="Browser(\\"h:fwd\\");"$ $MA+LIS,"[Go]",LM="Browser(GetStr(\\"\nURL> \\"));"$ ' + title_text + '\n\n'
if htm1.upper().find('<HTML') == -1:
return hb_header + htm1
dl_link = '$AN,"",A="BINARY"$Click $MA+LIS,"[Here]",LM="Get(\\"retry:send\\",\\"~/Downloads/' + l_url.split('/')[len(l_url.split('/'))-1] + '\\");"$ to download the file:\n\n' + l_url
if l_url.split('.')[len(l_url.split('.'))-1].upper() == 'Z':
dl_link = dl_link.replace('retry:send', 'retry:sendZ')
return hb_header + dl_link
htm1 = htm1[htm1.upper().find('<HTML'):]
htm1 = htm1.replace('$', '$$')
@ -41,7 +44,9 @@ def preprocess(htm1):
for f in soup1.findAll('a'):
for tag in f.findAll(True):
tag.decompose()
if str(tag).find('<None>') == -1:
if tag.name.upper() != 'IMG':
tag.decompose()
remove_tags = [ 'svg', 'embed', 'head', 'noscript', 'object', 'param', 'script', 'option' ]
for tag in remove_tags:
@ -59,10 +64,22 @@ def preprocess(htm1):
html = html.replace('</u>','$UL,0$')
html = html.replace('</U>','$UL,0$')
html = html.replace('<b>','$IV,1$')
html = html.replace('<B>','$IV,1$')
html = html.replace('</b>','$IV,0$')
html = html.replace('</B>','$IV,0$')
a_pos = html.upper().find('<IMG ')
while a_pos != -1:
html = html[:a_pos] + html[1+html.upper().find('>', a_pos):]
a_pos = html.upper().find('<IMG ')
img_text = html[a_pos:].split('>')[0]
img_text.replace("'",'"')
img_src = ''
img_pos = img_text.upper().find('SRC')
if img_pos > 0:
img_src = img_text[img_text.upper().find('SRC'):].split('"')[1]
img_el = '[URIEL_IMG]' + img_src + '[/URIEL_IMG]'
html = html[:a_pos] + img_el + html[1+html.upper().find('>', a_pos):]
a_pos = html.upper().find('<IMG ')
a_pos = html.upper().find('<BUTTON ')
while a_pos != -1:
@ -73,17 +90,23 @@ def preprocess(htm1):
html = html[:a_pos] + button_doctext + html[9+html.upper().find('</BUTTON>', a_pos):]
a_pos = html.upper().find('<BUTTON ')
a_ctr = 0
a_pos = html.upper().find('<A ')
while a_pos != -1:
link_pre = ''
link_text = html[a_pos:].split('>')[1]
link_text = link_text[:link_text.upper().find('</A')]
while link_text.find('[URIEL_IMG]') != -1:
link_pre += link_text[link_text.find('[URIEL_IMG]'):12+link_text.find('[/URIEL_IMG]')] + ' '
link_text = link_text[:link_text.find('[URIEL_IMG]')] + link_text[12+link_text.find('[/URIEL_IMG]'):]
link_text = link_text.replace('"','\\"')
link_href = ''
link_pos = html[a_pos:html.upper().find('</A>', a_pos)].upper().find('HREF')
if link_pos > 0:
link_href = html[a_pos:html.upper().find('</A>', a_pos)][link_pos:].replace('\'','"').split('"')[1]
doldoc_link = '$MA+LIS,"' + link_text + '",LM="Browser(\\"' + link_href + '\\");"$'
html = html[:a_pos] + doldoc_link + html[4+html.upper().find('</A>', a_pos):]
doldoc_link = '$AN,"",A="A' + str(a_ctr) + '"$$MA+LIS,"' + link_text + '",LM="Navigate(\\"A' + str(a_ctr) + '\\",\\"' + link_href + '\\");"$'
html = html[:a_pos] + link_pre + doldoc_link + html[4+html.upper().find('</A>', a_pos):]
a_ctr += 1
a_pos = html.upper().find('<A ')
a_pos = html.upper().find('<CENTER>')
@ -91,7 +114,10 @@ def preprocess(htm1):
center_text = html[a_pos:].split('>')[1]
center_text = center_text[:center_text.upper().find('</CENTER')]
center_text = center_text.replace('"','\\"')
center_doctext = '$TX+CX,"' + center_text + '"$'
if center_text.upper().find('[URIEL_IMG]') != -1:
center_doctext = center_text
else:
center_doctext = '$TX+CX,"' + center_text + '"$'
html = html[:a_pos] + center_doctext + html[9+html.upper().find('</CENTER>', a_pos):]
a_pos = html.upper().find('<CENTER>')
@ -142,6 +168,39 @@ def preprocess(htm1):
html = html.replace('&apos;','\'')
html = html.replace('&quot;','"')
img_a_ctr = 0
while html.find('[URIEL_IMG]') != -1:
img_url = html[11+html.find('[URIEL_IMG]'):html.find('[/URIEL_IMG]')]
img_ma = '$AN,"",A="IMG' + str(img_a_ctr) + '"$$MA+LIS,"[IMG]",LM="InsertImg(\\"IMG' + str(img_a_ctr) + '\\",\\"$IMIS$\\",\\"$IMIE$\\",\\"' + img_url + '.uriel_img\\");"$$AN,"",A="IMIS' + str(img_a_ctr) + '"$'
html = html[:html.find('[URIEL_IMG]')] + img_ma + html[12+html.find('[/URIEL_IMG]'):]
img_a_ctr += 1
hb_header = '$WW,1$$BLACK$$MA+LIS,"[Close]",LM="CloseBrowser;"$ $MA+LIS,"[Back]",LM="Browser(\\"h:back\\");"$ $MA+LIS,"[Fwd]",LM="Browser(\\"h:fwd\\");"$ $MA+LIS,"[Go]",LM="Browser(GetStr(\\"URL> \\"));"$ ' + title_text + '\n\n'
return hb_header + html
ind_id = ''
o_html = ''
o_lj_ct = 0
o_lj_indent = False
for line in html.split('\n'):
if not o_lj_indent:
if line[0:13] == '$AN,"",A="IMG':
# Left Justified image detected.
ind_id = line.split('IMG')[1].split('"')[0]
line = line.replace('$IMIS$','IMIS' + ind_id)
line = line.replace('$IMIE$','IMIE' + ind_id)
o_lj_indent = True
if o_lj_indent:
o_lj_ct += 1
if o_lj_ct > 11:
line = '$AN,"",A="IMIE' + ind_id + '"$' + line
ind_id = ''
o_lj_ct = 0
o_lj_indent = False
line = line.replace('$IMIS$','')
line = line.replace('$IMIE$','')
o_html += line + '\n'
return hb_header + o_html

34
uriel_proxy

@ -1,6 +1,6 @@
#!/usr/bin/python
from uriel_preprocessor import preprocess
import os, socket, subprocess, sys, time, urlparse
import os, socket, subprocess, sys, time, urlparse, uuid
HOST = '127.0.0.1'
PORT = 7202
@ -32,6 +32,7 @@ sys.stdout.flush()
while 1:
conn, addr = s.accept()
last_buf = ''
cmd_in = ''
rel_url = ''
history = []
@ -49,16 +50,39 @@ while 1:
if cmd_in.find('^') != -1:
get_file = cmd_in[cmd_in.find(DELIM_BIN_GET)+8:cmd_in.find('|')]
blk_ctr = 0
if get_file[0:2] == '//':
get_file = 'http:' + get_file
if get_file.find('://') != -1:
headers = { 'User-Agent': URIEL_VER_STR + ' (' + cmd_in.split('^')[0].rsplit('|')[1] + ')' }
file = subprocess.Popen('wget -O - -U "' + headers['User-Agent'] + '" "' + get_file + '" 2>/dev/null', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0]
if get_file.find('.uriel_img') != -1:
tmp_img_file = '/tmp/' + str(uuid.uuid4()) + '.bmp'
while os.path.exists(tmp_img_file):
tmp_img_file = '/tmp/' + str(uuid.uuid4()) + '.bmp'
file = subprocess.Popen('wget -O - -U "' + headers['User-Agent'] + '" "' + get_file.split('.uriel_img')[0] + '" 2>/dev/null | gm convert -resize 100x100 - -colors 16 "' + tmp_img_file + '"', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0]
file = open(tmp_img_file, "rb").read()
os.remove(tmp_img_file)
else:
file = subprocess.Popen('wget -O - -U "' + headers['User-Agent'] + '" "' + get_file + '" 2>/dev/null', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0]
else:
file = open(get_file, "rb").read()
file = ''
if get_file == 'retry:send':
file = last_buf
if get_file == 'retry:sendZ':
tmp_z_file = '/tmp/' + str(uuid.uuid4()) + '.Z'
while os.path.exists(tmp_z_file):
tmp_z_file = '/tmp/' + str(uuid.uuid4()) + '.Z'
open(tmp_z_file, "wb").write(last_buf)
z = subprocess.Popen('tosz "' + tmp_z_file + '"', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0]
file = open(tmp_z_file.split('.Z')[0], "rb").read()
os.remove(tmp_z_file.split('.Z')[0])
if file == '':
file = open(get_file, "rb").read()
while blk_ctr < len(file):
conn.sendall(file[blk_ctr:blk_ctr+blk_size])
blk_ctr += blk_size
time.sleep(delay_ms)
conn.sendall(DELIM_BIN_EOF+'\x00')
last_buf = file
cmd_in = ""
state = MODE_LISTEN
@ -109,7 +133,9 @@ while 1:
url = url[:url.find('//')] + '//' + url[url.find('//')+2:].replace('//','/')
headers = { 'User-Agent': user_agent }
data = subprocess.Popen('wget -O - -U "' + headers['User-Agent'] + '" "' + url + '" 2>/dev/null', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0]
page = preprocess(data)
page = preprocess(data, url)
if page.find('$AN,"",A="BINARY"$') != -1:
last_buf = data
hst_index += 1
history = history[0:hst_index]
history.append({'url':url, 'page':page})

Loading…
Cancel
Save