module.exports = function(redis) { const config = require('../config') const https = require('https') const parser = require('node-html-parser') const fs = require('fs') const path = require('path') this.download = async (url, params = '') => { return new Promise(resolve => { if(!url) { resolve({ success: false, reason: 'MISSING_URL' }) } if(url.includes('?')) { let wikipage = url.split('wikipedia.org/wiki/')[1] let uriencoded_wikipage = '' if(wikipage) { uriencoded_wikipage = encodeURIComponent(wikipage) } url = url.replace(wikipage, uriencoded_wikipage) } if(params) { url = `${url}?${params}` } redis.get(url, (error, data) => { if(data) { console.log(`Got key ${url} from cache.`) resolve({ success: true, html: data, processed: true }) } else { https.get(url, (res) => { if(res.statusCode !== 200) { if(res.statusCode === 301 || res.statusCode === 302) { resolve({ success: false, reason: 'REDIRECT', url: res.headers.location }) res.resume() } else if(res.statusCode === 404) { // let's redirect 404s to homepage console.log(`Didn't find ${url} HTTP status code: ${res.statusCode}`) res.resume() return resolve({ success: false, reason: 'REDIRECT', url: 'https://wikipedia.org/' }) } else { console.log(`Error while fetching data from ${url}. HTTP status code: ${res.statusCode}`) res.resume() return resolve({ success: false, reason: `INVALID_HTTP_RESPONSE: ${res.statusCode}` }) } } let data = '' res.on('data', (chunk) => { data += chunk }) res.on('end', () => { console.log(`Fetched url ${url} from Wikipedia.`) resolve({ success: true, html: data, processed: false }) }) }).on('error', (err) => { console.log(`Error while fetching data from ${url}. Error: ${err}`) resolve({ success: false, reason: 'SERVER_ERROR' }) }) } }) }) } this.applyUserMods = (data, theme, lang) => { /** * We have already processed the HTML, but we haven't applied the user's * cookie specific modifications to it yet. Let's do it. */ // load fr style, issue #12 const langSuffix = `_${lang}` ? lang === 'fr' : '' if(theme === 'white') { // if the user has chosen the white theme from the preferences data = data.replace('', ``) } else if(theme === 'dark') { // if the user has chosen the dark theme from the preferences data = data.replace('', ` `) } else { // default, auto theme data = data.replace('', ``) } return data } this.processHtml = (data, url, params, lang, user_preferences, process_user_prefs) => { return new Promise(resolve => { if(validHtml(data.html)) { url = encodeURI(url) if(params) { url = `${url}?${params}` } data.html = parser.parse(data.html) // replace default wikipedia top right nav bar links let nav = data.html.querySelector('nav#p-personal .vector-menu-content-list') if(nav) { nav.innerHTML = `
  • [ about ]
  • [ preferences ]
  • ` } // append the lang query param to the URLs starting with /wiki/ or /w/ let links = data.html.querySelectorAll('a') for(let i = 0; i < links.length; i++) { let href = links[i].getAttribute('href') if(href && (href.startsWith('/wiki/') || href.startsWith('/w/'))) { href = `${protocol}${config.domain}${href}` let u = new URL(href) u.searchParams.append('lang', lang) href = `${u.pathname}${u.search}` links[i].setAttribute('href', href) } } // remove #p-wikibase-otherprojects let wikibase_links = data.html.querySelector('#p-wikibase-otherprojects') if(wikibase_links) { wikibase_links.remove() } // remove all