redlib-org/redlib-instances
1
0
Fork 0
mirror of https://github.com/redlib-org/redlib-instances.git synced 2024-11-13 18:06:15 -05:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

allow for use of custom User-Agent

Browse Source
This commit is contained in:
Daniel Valentine 2022-09-26 20:42:09 -06:00
parent ecca464ec7
commit 3f696d479c
1 changed files with 18 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
generate-instances-json.sh
View File

@ -32,6 +32,10 @@ TODAY="$(date -I -u)"
# jq is required for JSON processing. # jq is required for JSON processing.
DEPENDENCIES=(curl jq) DEPENDENCIES=(curl jq)
# If USER_AGENT is specified in the envs, we'll pass this argument to curl
# using the -A flag to set a custom User-Agent.
USER_AGENT="${USER_AGENT:-}"
# check_tor # check_tor
# #
# Returns true if tor is running; false otherwise. # Returns true if tor is running; false otherwise.
@ -341,12 +345,18 @@ get ()
return 103 return 103
fi fi
# Use a custom User-Agent if provided.
if [[ -n "${USER_AGENT?}" ]]
then
curl_cmd=("${curl_cmd[@]}" -A "${USER_AGENT}")
fi
# Do the GET. Try up to the number of times specified in the tries variable. # Do the GET. Try up to the number of times specified in the tries variable.
for (( i = tries; i > 0; i-- )) for (( i = tries; i > 0; i-- ))
do do
"${curl_cmd[@]}" -m"${timeout}" -fsL -- "${scheme}://${url_no_scheme}" "${curl_cmd[@]}" -m"${timeout}" -fsL -- "${scheme}://${url_no_scheme}"
rc=$? rc=$?
if [[ ${rc} -eq 0 ]] if [[ ${rc} -eq 0 ]]
then then
return return
@ -482,7 +492,7 @@ create_instance_entry ()
# helpdoc # helpdoc
# #
# TODO # Print usage information to stdout.
helpdoc () helpdoc ()
{ {
cat <<! cat <<!
@ -538,6 +548,11 @@ OPTIONS
Write the results to OUTPUT_JSON. Any existing file will be Write the results to OUTPUT_JSON. Any existing file will be
overwritten. To write to stdout (the default behavior), either omit overwritten. To write to stdout (the default behavior), either omit
this option or provide \`-o -\`. this option or provide \`-o -\`.
ENVIRONMENT
USER_AGENT
Sets the User-Agent that curl will use when making the GET to each website.
! !
} }