module.exports = function(redis) {
const config = require('../config')
const https = require('https')
const parser = require('node-html-parser')
const fs = require('fs')
const path = require('path')
this.download = async (url, params = '') => {
return new Promise(resolve => {
if(!url) {
resolve({ success: false, reason: 'MISSING_URL' })
}
url = encodeURI(url)
if(params != '') {
url = `${url}?${params}`
}
redis.get(url, (error, data) => {
if(data) {
console.log(`Got key ${url} from cache.`)
resolve({ success: true, html: data, processed: true })
} else {
https.get(url, (res) => {
if(res.statusCode !== 200) {
if(res.statusCode === 301 || res.statusCode === 302) {
resolve({ success: false, reason: 'REDIRECT', url: res.headers.location })
} else if(res.statusCode === 404) {
// let's redirect 404s to homepage
console.log(`Didn't find ${url} HTTP status code: ${res.statusCode}`)
return resolve({ success: false, reason: 'REDIRECT', url: 'https://wikipedia.org/' })
} else {
console.log(`Error while fetching data from ${url}. HTTP status code: ${res.statusCode}`)
return resolve({ success: false, reason: `INVALID_HTTP_RESPONSE: ${res.statusCode}` })
}
}
let data = ''
res.on('data', (chunk) => {
data += chunk
})
res.on('end', () => {
console.log(`Fetched url ${url} from Wikipedia.`)
resolve({ success: true, html: data, processed: false })
})
}).on('error', (err) => {
console.log(`Error while fetching data from ${url}. Error: ${err}`)
resolve({ success: false, reason: 'SERVER_ERROR' })
})
}
})
})
}
this.applyUserMods = (data, user_preferences) => {
/**
* We have already processed the HTML, but we haven't applied the user's
* cookie specific modifications to it yet. Let's do it.
*/
// add dark style theme file if needed
if(user_preferences.theme == 'dark') {
data = data.replace('', ``)
}
return data
}
this.processHtml = (data, url, params, lang, user_preferences, process_user_prefs) => {
return new Promise(resolve => {
if(validHtml(data.html)) {
url = encodeURI(url)
if(params != '') {
url = `${url}?${params}`
}
let setting_lang = user_preferences.default_lang || config.default_lang
data.html = parser.parse(data.html)
// insert wikiless styles
let head = data.html.querySelector('head')
if(head) {
let head_html = ``
head.insertAdjacentHTML('beforeend', head_html)
}
// replace default wikipedia top right nav bar links
let nav = data.html.querySelector('nav#p-personal .vector-menu-content-list')
if(nav) {
nav.innerHTML = `
[ about ]
[ preferences ]
`
}
// append the lang query param to the URLs starting with /wiki/ or /w/
let links = data.html.querySelectorAll('a')
for(let i = 0; i < links.length; i++) {
let href = links[i].getAttribute('href')
if(href) {
if(href.startsWith('/wiki/') || href.startsWith('/w/')) {
href = `${protocol}${config.domain}${href}`
let u = new URL(href)
u.searchParams.append('lang', lang)
href = u.href
links[i].setAttribute('href', href)
}
}
}
// remove #p-wikibase-otherprojects
let wikibase_links = data.html.querySelector('#p-wikibase-otherprojects')
if(wikibase_links) {
wikibase_links.remove()
}
/**
* Remove the language subdomain from the sidebar language switchers.
* Then append the language as a URL query param.
*/
let lang_links = data.html.querySelectorAll('#p-lang .interlanguage-link a')
for(let i = 0; i < lang_links.length; i++) {
let href = lang_links[i].getAttribute('href')
let lang_code = href.split('wikipedia.org')[0].split('//')[1]
href = href.replace(lang_code, '')
href = `${href}?lang=${lang_code.slice(0, -1)}`
lang_links[i].setAttribute('href', href)
}
data.html = data.html.toString()
// replace upload.wikimedia.org with /media
const upload_wikimedia_regx = /((https:|http:|)\/\/?upload.wikimedia.org)/gm
data.html = data.html.replace(upload_wikimedia_regx, '/media')
// replace maps.wikimedia.org with /media
const maps_wikimedia_regx = /((https:|http:|)\/\/?maps.wikimedia.org)/gm
data.html = data.html.replace(maps_wikimedia_regx, '/media/maps_wikimedia_org')
// replace wiki links
const wiki_href_regx = /(href=\"(https:|http:|)\/\/([A-z.]+\.)?(wikipedia.org|wikimedia.org|wikidata.org|mediawiki.org))/gm
data.html = data.html.replace(wiki_href_regx, `href="${protocol}${config.domain}`)
redis.setex(url, config.setexs.wikipage, data.html, (error) => {
if(error) {
console.log(`Error setting the ${url} key to Redis. Error: ${error}`)
resolve({ success: false, reason: 'SERVER_ERROR_REDIS_SET' })
} else {
resolve({ success: true, html: data.html })
}
})
} else {
console.log('Invalid wiki_html.')
resolve({ success: false, reason: 'INVALID_HTML' })
}
})
}
this.proxyMedia = (req, wiki_domain='') => {
return new Promise(async resolve => {
let params = new URLSearchParams(req.query).toString() || ''
if(params != '') {
params = '?' + params
}
let path = ''
let domain = 'upload.wikimedia.org'
let wikimedia_path = ''
if(wiki_domain === 'maps.wikimedia.org') {
path = req.url.split('/media/maps_wikimedia_org')[1]
domain = 'maps.wikimedia.org'
wikimedia_path = path
} else {
path = req.url.split('/media')[1]
wikimedia_path = path + params
}
url = new URL(`https://${domain}${wikimedia_path}`)
let file = await saveFile(url, path)
if(file.success === true) {
resolve({ success: true, path: file.path })
} else {
resolve({ success: false, reason: file.reason })
}
})
}
this.saveFile = (url, file_path) => {
return new Promise(async resolve => {
let media_path = ''
if(url.href.startsWith('https://maps.wikimedia.org/')) {
media_path = path.join(__dirname, '../media/maps_wikimedia_org')
} else {
media_path = path.join(__dirname, '../media')
}
const path_with_filename = decodeURI(`${media_path}${file_path}`)
let path_without_filename = path_with_filename.split('/')
path_without_filename.pop()
path_without_filename = path_without_filename.join('/')
if(!fs.existsSync(path_with_filename)) {
fs.mkdir(path_without_filename, { recursive: true }, (err) => {
if(err) {
resolve({ success: false, reason: 'MKDIR_FAILED' })
} else {
let file = fs.createWriteStream(path_with_filename)
https.get(url.href, (res) => {
res.pipe(file)
file.on('finish', () => {
file.close()
resolve({ success: true, path: path_with_filename })
})
}).on('error', (err) => {
console.log('Error while fetching data. Details:', err)
resolve({ success: false, reason: 'SERVER_ERROR' })
})
}
})
} else {
resolve({ success: true, path: path_with_filename })
}
})
}
this.writeToDisk = (data, file) => {
return new Promise(resolve => {
fs.writeFile(file, data, 'binary', (error, result) => {
if(!error) {
resolve({ success: true })
} else {
resolve({ success: false, reason: error })
}
})
}).catch((err) => {
console.log('Writing media file to disk failed for unknown reason. Details:', err)
})
}
this.handleWikiPage = async (req, res, prefix) => {
let lang = req.query.lang || req.cookies.default_lang || config.default_lang
if(lang) {
if(typeof(lang) !== 'string') {
lang = lang[0]
} else {
lang = lang.split('?')[0]
}
}
if(!validLang(lang)) {
return res.send('invalid lang')
}
let params = new URLSearchParams(req.query).toString()
let url = ''
if(prefix === '/wiki/') {
let page = req.params.page || ''
let sub_page = req.params.sub_page || ''
if(sub_page != '') {
sub_page = `/${sub_page}`
}
url = `https://${lang}.wikipedia.org/wiki/${page}${sub_page}`
}
if(prefix === '/w/') {
let file = req.params.file
url = `https://${lang}.wikipedia.org/w/${file}`
}
if(prefix === '/wiki/Map') {
let page = 'Special:Map'
let sub_page = req.params['0'] || ''
url = `https://${lang}.wikipedia.org/wiki/${page}/${sub_page}`
}
if(prefix === '/') {
url = `https://${lang}.wikipedia.org/?lang=${lang}`
}
let result = await download(url, params)
if(result.success !== true) {
if(result.reason === 'REDIRECT' && result.url != '') {
url = result.url.split('wikipedia.org')[1]
let prefix = ''
if(url) {
if(url.startsWith('/w/')) {
prefix = '/w/'
}
if(url.startsWith('/wiki/')) {
prefix = '/wiki/'
}
}
if(prefix != '') {
redirect_to = `${prefix}${result.url.split(prefix)[1]}`
return res.redirect(redirect_to)
} else {
return res.redirect(`/?lang=${lang}`)
}
}
}
if(result.processed === true) {
return res.send(applyUserMods(result.html, req.cookies))
} else {
let process_html = await processHtml(result, url, params, lang, req.cookies)
if(process_html.success === true) {
return res.send(applyUserMods(process_html.html.toString(), req.cookies))
} else {
return res.send(process_html.reason)
}
}
}
this.validLang = (lang, return_langs=false) => {
let valid_langs = ['ab','ace','ady','af','ak','als','am','an','ang','ar',
'arc','ary','arz','as','ast','atj','av','avk','awa','ay','az','azb','ba',
'ban','bar','bat-smg','bcl','be','be-tarask','bg','bh','bi','bjn','bm','bn',
'bo','bpy','br','bs','bug','bxr','ca','cbk-zam','cdo','ce','ceb','ch','chr',
'chy','ckb','co','cr','crh','cs','csb','cu','cv','cy','da','de','din','diq',
'dsb','dty','dv','dz','ee','el','eml','en','eo','es','et','eu','ext','fa',
'ff','fi','fiu-vro','fj','fo','fr','frp','frr','fur','fy','ga','gag','gan',
'gcr','gd','gl','glk','gn','gom','gor','got','gu','gv','ha','hak','haw',
'he','hi','hif','hr','hsb','ht','hu','hy','hyw','ia','id','ie','ig','ik',
'ilo','inh','io','is','it','iu','ja','jam','jbo','jv','ka','kaa','kab',
'kbd','kbp','kg','ki','kk','kl','km','kn','ko','koi','krc','ks','ksh','ku',
'kv','kw','ky','la','lad','lb','lbe','lez','lfn','lg','li','lij','lld',
'lmo','ln','lo','lt','ltg','lv','mad','mai','map-bms','mdf','mg','mhr','mi',
'min','mk','ml','mn','mnw','mr','mrj','ms','mt','mwl','my','myv','mzn','na',
'nah','nap','nds','nds-nl','ne','new','nia','nl','nn','no','nostalgia',
'nov','nqo','nrm','nso','nv','ny','oc','olo','om','or','os','pa','pag',
'pam','pap','pcd','pdc','pfl','pi','pih','pl','pms','pnb','pnt','ps','pt',
'qu','rm','rmy','rn','ro','roa-rup','roa-tara','ru','rue','rw','sa','sah',
'sat','sc','scn','sco','sd','se','sg','sh','shn','si','simple','sk','skr',
'sl','sm','smn','sn','so','sq','sr','srn','ss','st','stq','su','sv','sw',
'szl','szy','ta','tcy','te','tet','tg','th','ti','tk','tl','tn','to','tpi',
'tr','ts','tt','tum','tw','ty','tyv','udm','ug','uk','ur','uz','ve','vec',
'vep','vi','vls','vo','wa','war','wo','wuu','xal','xh','xmf','yi','yo','za',
'zea','zh','zh-classical','zh-min-nan','zh-yue','zu']
if(return_langs) {
return valid_langs
}
if(valid_langs.includes(lang)) {
return true
}
return false
}
this.validHtml = (html) => {
if(html) {
if(parser.valid(html)) {
return true
}
}
return false
}
this.wikilessLogo = () => {
let static_path = path.join(__dirname, '../static')
return `${static_path}/wikiless-logo.png`
}
this.preferencesPage = (req, res) => {
let user_preferences = req.cookies
let lang_select = ''
let back = req.url.split('?back=')[1]
let html = `
Preferences - Wikiless