module.exports = function(redis) { const config = require('../config') const parser = require('node-html-parser') const fs = require('fs').promises const { createWriteStream, existsSync } = require('fs') const path = require('path') const got = require('got') const stream = require('stream') const { promisify } = require('util') const pipeline = promisify(stream.pipeline) this.download = async (url, params = '') => { if(!url) { return { success: false, reason: 'MISSING_URL' } } if(url.includes('?')) { let wikipage = url.split('wikipedia.org/wiki/')[1] let uriencoded_wikipage = '' if(wikipage) { uriencoded_wikipage = encodeURIComponent(wikipage) } url = url.replace(wikipage, uriencoded_wikipage) } if(params) { url = `${url}?${params}` } let data = '' try { if(redis.isOpen === false) { await redis.connect() } data = await redis.get(url) } catch(err) { console.log(`Error while fetching key ${url} from cache. Error: ${err}`) } if(data) { console.log(`Got key ${url} from cache.`) return { success: true, html: data, processed: true } } try { const gotUrl = new URL(url) const { body } = await got(gotUrl) console.log(`Fetched url ${url} from Wikipedia.`) return { success: true, html: body, processed: false } } catch(err) { const { statusCode } = err.response if(statusCode !== 200) { if(statusCode === 404) { // let's redirect 404s to homepage console.log(`Didn't find ${url} HTTP status code: ${statusCode}`) return { success: false, reason: 'REDIRECT', url: 'https://wikipedia.org/' } } console.log(`Error while fetching data from ${url}. HTTP status code: ${statusCode}`) return { success: false, reason: `INVALID_HTTP_RESPONSE: ${statusCode}` } } } } this.applyUserMods = (data, theme, lang) => { /** * We have already processed the HTML, but we haven't applied the user's * cookie specific modifications to it yet. Let's do it. */ // load fr style, issue #12 const langSuffix = (lang === 'fr') ? `_${lang}` : ''; if(theme === 'white') { // if the user has chosen the white theme from the preferences data = data.replace('', ``) } else if(theme === 'dark') { // if the user has chosen the dark theme from the preferences data = data.replace('', ` `) } else { // default, auto theme data = data.replace('', ``) } return data } this.processHtml = async (data, url, params, lang) => { if(validHtml(data.html)) { const decoded_url = url url = encodeURI(url) if(params) { url = `${url}?${params}` } data.html = parser.parse(data.html) // replace default wikipedia top right nav bar links let nav = data.html.querySelector('nav#p-personal .vector-menu-content-list') if(nav) { nav.innerHTML = `