Metastem/wikiless
1
0
Fork 0
mirror of https://github.com/Metastem/wikiless synced 2024-12-06 19:16:58 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

consume response data from the https.get() requests

Browse Source
This commit is contained in:
orenom
2021-07-06 13:46:03 +02:00
parent 22a465eb35
commit 37887cd890
1 changed files with 12 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils.js
+12 -8
View File
@@ -12,25 +12,28 @@ module.exports = function(redis) {
}
url = encodeURI(url)
if(params != '') {
if(params != '') {
url = `${url}?${params}`
}
redis.get(url, (error, data) => {
if(data) {
console.log(`Got key ${url} from cache.`)
console.log(`Got key ${url} from cache.`)
resolve({ success: true, html: data, processed: true })
} else {
https.get(url, (res) => {
if(res.statusCode !== 200) {
if(res.statusCode === 301 || res.statusCode === 302) {
resolve({ success: false, reason: 'REDIRECT', url: res.headers.location })
resolve({ success: false, reason: 'REDIRECT', url: res.headers.location })
res.resume()
} else if(res.statusCode === 404) {
// let's redirect 404s to homepage
console.log(`Didn't find ${url} HTTP status code: ${res.statusCode}`)
return resolve({ success: false, reason: 'REDIRECT', url: 'https://wikipedia.org/' })
res.resume()
return resolve({ success: false, reason: 'REDIRECT', url: 'https://wikipedia.org/' })
} else {
console.log(`Error while fetching data from ${url}. HTTP status code: ${res.statusCode}`)
res.resume()
return resolve({ success: false, reason: `INVALID_HTTP_RESPONSE: ${res.statusCode}` })
}
}
@@ -71,7 +74,7 @@ module.exports = function(redis) {
return new Promise(resolve => {
if(validHtml(data.html)) {
url = encodeURI(url)
if(params != '') {
if(params != '') {
url = `${url}?${params}`
}
@@ -165,7 +168,7 @@ module.exports = function(redis) {
this.proxyMedia = (req, wiki_domain='') => {
return new Promise(async resolve => {
let params = new URLSearchParams(req.query).toString() || ''
let params = new URLSearchParams(req.query).toString() || ''
if(params != '') {
params = '?' + params
@@ -201,7 +204,7 @@ module.exports = function(redis) {
let media_path = ''
if(url.href.startsWith('https://maps.wikimedia.org/')) {
media_path = path.join(__dirname, '../media/maps_wikimedia_org')
} else {
} else {
media_path = path.join(__dirname, '../media')
}
@@ -224,6 +227,7 @@ module.exports = function(redis) {
res.pipe(file)
file.on('finish', () => {
file.close()
res.resume()
resolve({ success: true, path: path_with_filename })
})
}).on('error', (err) => {
@@ -253,7 +257,7 @@ module.exports = function(redis) {
}
this.handleWikiPage = async (req, res, prefix) => {
let lang = req.query.lang || req.cookies.default_lang || config.default_lang
let lang = req.query.lang || req.cookies.default_lang || config.default_lang
if(lang) {
if(typeof(lang) !== 'string') {